From 1d24aa0e0fcaff47916f414302a7cea20f43ffea Mon Sep 17 00:00:00 2001 From: "shijia.wxr" Date: Fri, 14 Nov 2014 17:19:43 +0800 Subject: [PATCH] =?UTF-8?q?=E9=87=8D=E6=96=B0=E5=BC=80=E6=BA=90=EF=BC=8Cv3?= =?UTF-8?q?.2.2=EF=BC=8C=E4=BB=8D=E7=84=B6=E4=B8=8E=E9=98=BF=E9=87=8C?= =?UTF-8?q?=E5=86=85=E9=83=A8=E7=89=88=E6=9C=AC=E4=BF=9D=E6=8C=81=E4=B8=80?= =?UTF-8?q?=E8=87=B4=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 49 + .travis.yml | 1 + LICENSE.txt | 191 + README.md | 42 +- benchmark/consumer.sh | 6 + benchmark/producer.sh | 6 + benchmark/runclass.sh | 30 + benchmark/tproducer.sh | 6 + bin/README.md | 34 + bin/mqadmin | 34 + bin/mqadmin.xml | 26 + bin/mqbroker | 34 + bin/mqbroker.xml | 26 + bin/mqfiltersrv | 34 + bin/mqfiltersrv.xml | 26 + bin/mqnamesrv | 34 + bin/mqnamesrv.xml | 26 + bin/mqshutdown | 34 + bin/os.sh | 40 + bin/play.sh | 18 + bin/runbroker.sh | 38 + bin/runserver.sh | 32 + bin/startfsrv.sh | 34 + bin/tools.sh | 28 + conf/2m-2s-async/broker-a-s.properties | 7 + conf/2m-2s-async/broker-a.properties | 7 + conf/2m-2s-async/broker-b-s.properties | 7 + conf/2m-2s-async/broker-b.properties | 7 + conf/2m-2s-sync/broker-a-s.properties | 7 + conf/2m-2s-sync/broker-a.properties | 7 + conf/2m-2s-sync/broker-b-s.properties | 7 + conf/2m-2s-sync/broker-b.properties | 7 + conf/2m-noslave/broker-a.properties | 7 + conf/2m-noslave/broker-b.properties | 7 + conf/logback_broker.xml | 217 + conf/logback_filtersrv.xml | 70 + conf/logback_namesrv.xml | 70 + conf/logback_tools.xml | 76 + deploy.bat | 1 + develop.sh | 13 + docs/rocketmq.java.code.style.xml | 432 ++ eclipse.bat | 1 + install.bat | 2 + install.sh | 11 + pom.xml | 365 + release-client.xml | 33 + release.xml | 43 + rocketmq-broker/pom.xml | 65 + .../rocketmq/broker/BrokerController.java | 837 +++ .../broker/BrokerPathConfigHelper.java | 35 + .../rocketmq/broker/BrokerStartup.java | 285 + .../broker/client/ClientChannelInfo.java | 117 + .../client/ClientHousekeepingService.java | 108 + .../broker/client/ConsumerGroupInfo.java | 281 + .../client/ConsumerIdsChangeListener.java | 29 + .../broker/client/ConsumerManager.java | 202 + .../DefaultConsumerIdsChangeListener.java | 47 + .../broker/client/ProducerManager.java | 201 + .../broker/client/net/Broker2Client.java | 316 + .../rebalance/RebalanceLockManager.java | 312 + .../broker/filtersrv/FilterServerManager.java | 174 + .../broker/filtersrv/FilterServerUtil.java | 55 + .../broker/longpolling/ManyPullRequest.java | 51 + .../broker/longpolling/PullRequest.java | 70 + .../longpolling/PullRequestHoldService.java | 172 + .../broker/mqtrace/ConsumeMessageContext.java | 132 + .../broker/mqtrace/ConsumeMessageHook.java | 26 + .../broker/mqtrace/SendMessageContext.java | 176 + .../broker/mqtrace/SendMessageHook.java | 26 + .../broker/offset/ConsumerOffsetManager.java | 265 + .../rocketmq/broker/out/BrokerOuterAPI.java | 348 + .../broker/pagecache/ManyMessageTransfer.java | 97 + .../broker/pagecache/OneMessageTransfer.java | 86 + .../pagecache/QueryMessageTransfer.java | 97 + .../processor/AdminBrokerProcessor.java | 1232 ++++ .../processor/ClientManageProcessor.java | 350 + .../processor/EndTransactionProcessor.java | 255 + .../processor/ForwardRequestProcessor.java | 51 + .../processor/PullMessageProcessor.java | 515 ++ .../processor/QueryMessageProcessor.java | 173 + .../processor/SendMessageProcessor.java | 669 ++ .../broker/slave/SlaveSynchronize.java | 166 + .../SubscriptionGroupManager.java | 172 + .../broker/topic/TopicConfigManager.java | 473 ++ .../broker/transaction/TransactionRecord.java | 27 + .../broker/transaction/TransactionStore.java | 32 + .../jdbc/JDBCTransactionStore.java | 263 + .../jdbc/JDBCTransactionStoreConfig.java | 48 + .../src/main/resources/transaction.sql | 4 + .../rocketmq/broker/api/SendMessageTest.java | 73 + .../offset/ConsumerOffsetManagerTest.java | 52 + .../broker/topic/TopicConfigManagerTest.java | 55 + .../jdbc/JDBCTransactionStoreTest.java | 94 + rocketmq-client/pom.xml | 25 + .../alibaba/rocketmq/client/ClientConfig.java | 160 + .../com/alibaba/rocketmq/client/MQAdmin.java | 144 + .../com/alibaba/rocketmq/client/MQHelper.java | 97 + .../alibaba/rocketmq/client/QueryResult.java | 55 + .../alibaba/rocketmq/client/Validators.java | 137 + .../rocketmq/client/VirtualEnvUtil.java | 63 + .../client/admin/MQAdminExtInner.java | 24 + .../AllocateMessageQueueStrategy.java | 55 + .../consumer/DefaultMQPullConsumer.java | 352 + .../consumer/DefaultMQPushConsumer.java | 459 ++ .../rocketmq/client/consumer/MQConsumer.java | 67 + .../client/consumer/MQPullConsumer.java | 150 + .../MQPullConsumerScheduleService.java | 245 + .../client/consumer/MQPushConsumer.java | 106 + .../client/consumer/MessageQueueListener.java | 32 + .../client/consumer/PullCallback.java | 29 + .../rocketmq/client/consumer/PullResult.java | 84 + .../rocketmq/client/consumer/PullStatus.java | 39 + .../client/consumer/PullTaskCallback.java | 23 + .../client/consumer/PullTaskContext.java | 45 + .../listener/ConsumeConcurrentlyContext.java | 73 + .../listener/ConsumeConcurrentlyStatus.java | 29 + .../listener/ConsumeOrderlyContext.java | 70 + .../listener/ConsumeOrderlyStatus.java | 35 + .../consumer/listener/MessageListener.java | 26 + .../listener/MessageListenerConcurrently.java | 42 + .../listener/MessageListenerOrderly.java | 42 + .../AllocateMessageQueueAveragely.java | 79 + ...AllocateMessageQueueAveragelyByCircle.java | 74 + .../AllocateMessageQueueByConfig.java | 55 + .../AllocateMessageQueueByMachineRoom.java | 80 + .../consumer/store/LocalFileOffsetStore.java | 235 + .../store/OffsetSerializeWrapper.java | 44 + .../client/consumer/store/OffsetStore.java | 71 + .../client/consumer/store/ReadOffsetType.java | 25 + .../store/RemoteBrokerOffsetStore.java | 262 + .../client/exception/MQBrokerException.java | 50 + .../client/exception/MQClientException.java | 56 + .../client/hook/CheckForbiddenContext.java | 150 + .../client/hook/CheckForbiddenHook.java | 17 + .../client/hook/ConsumeMessageContext.java | 103 + .../client/hook/ConsumeMessageHook.java | 26 + .../client/hook/FilterMessageContext.java | 93 + .../client/hook/FilterMessageHook.java | 29 + .../client/hook/SendMessageContext.java | 137 + .../rocketmq/client/hook/SendMessageHook.java | 26 + .../client/impl/ClientRemotingProcessor.java | 244 + .../client/impl/CommunicationMode.java | 28 + .../client/impl/FindBrokerResult.java | 41 + .../rocketmq/client/impl/MQAdminImpl.java | 404 ++ .../rocketmq/client/impl/MQClientAPIImpl.java | 2407 +++++++ .../rocketmq/client/impl/MQClientManager.java | 77 + .../ConsumeMessageConcurrentlyService.java | 460 ++ .../ConsumeMessageOrderlyService.java | 551 ++ .../impl/consumer/ConsumeMessageService.java | 58 + .../consumer/DefaultMQPullConsumerImpl.java | 718 ++ .../consumer/DefaultMQPushConsumerImpl.java | 1192 +++ .../client/impl/consumer/MQConsumerInner.java | 66 + .../impl/consumer/MessageQueueLock.java | 46 + .../client/impl/consumer/ProcessQueue.java | 452 ++ .../client/impl/consumer/PullAPIWrapper.java | 329 + .../impl/consumer/PullMessageService.java | 133 + .../client/impl/consumer/PullRequest.java | 115 + .../client/impl/consumer/PullResultExt.java | 57 + .../client/impl/consumer/RebalanceImpl.java | 533 ++ .../impl/consumer/RebalancePullImpl.java | 88 + .../impl/consumer/RebalancePushImpl.java | 212 + .../impl/consumer/RebalanceService.java | 60 + .../client/impl/factory/MQClientInstance.java | 1300 ++++ .../impl/producer/DefaultMQProducerImpl.java | 1058 +++ .../client/impl/producer/MQProducerInner.java | 51 + .../impl/producer/TopicPublishInfo.java | 112 + .../rocketmq/client/log/ClientLogger.java | 117 + .../client/producer/DefaultMQProducer.java | 333 + .../producer/LocalTransactionExecuter.java | 29 + .../producer/LocalTransactionState.java | 30 + .../rocketmq/client/producer/MQProducer.java | 216 + .../client/producer/MessageQueueSelector.java | 32 + .../client/producer/SendCallback.java | 29 + .../rocketmq/client/producer/SendResult.java | 99 + .../rocketmq/client/producer/SendStatus.java | 33 + .../producer/TransactionCheckListener.java | 29 + .../producer/TransactionMQProducer.java | 116 + .../producer/TransactionSendResult.java | 40 + .../selector/SelectMessageQueueByHash.java | 43 + .../SelectMessageQueueByMachineRoom.java | 51 + .../selector/SelectMessageQueueByRandoom.java | 46 + .../client/stat/ConsumerStatsManager.java | 155 + .../main/resources/log4j_rocketmq_client.xml | 41 + .../resources/logback_rocketmq_client.xml | 41 + .../client/TestMultiConsumerProducer.java | 72 + .../rocketmq/client/ValidatorsTest.java | 23 + .../AllocateMessageQueueAveragelyTest.java | 277 + rocketmq-common/pom.xml | 26 + .../alibaba/rocketmq/common/BrokerConfig.java | 375 + .../rocketmq/common/ConfigManager.java | 102 + .../alibaba/rocketmq/common/DataVersion.java | 70 + .../alibaba/rocketmq/common/MQVersion.java | 238 + .../com/alibaba/rocketmq/common/MixAll.java | 484 ++ .../com/alibaba/rocketmq/common/Pair.java | 50 + .../alibaba/rocketmq/common/ServiceState.java | 40 + .../rocketmq/common/ServiceThread.java | 159 + .../alibaba/rocketmq/common/SystemClock.java | 72 + .../rocketmq/common/ThreadFactoryImpl.java | 22 + .../alibaba/rocketmq/common/TopicConfig.java | 195 + .../rocketmq/common/TopicFilterType.java | 34 + .../com/alibaba/rocketmq/common/UtilAll.java | 466 ++ .../rocketmq/common/admin/ConsumeStats.java | 69 + .../rocketmq/common/admin/OffsetWrapper.java | 59 + .../rocketmq/common/admin/RollbackStats.java | 76 + .../rocketmq/common/admin/TopicOffset.java | 58 + .../common/admin/TopicStatsTable.java | 42 + .../common/annotation/ImportantField.java | 29 + .../rocketmq/common/constant/LoggerName.java | 34 + .../rocketmq/common/constant/PermName.java | 59 + .../common/consumer/ConsumeFromWhere.java | 47 + .../rocketmq/common/filter/FilterAPI.java | 107 + .../rocketmq/common/filter/MessageFilter.java | 18 + .../rocketmq/common/filter/impl/Op.java | 24 + .../rocketmq/common/filter/impl/Operand.java | 12 + .../rocketmq/common/filter/impl/Operator.java | 62 + .../common/filter/impl/PolishExpr.java | 198 + .../rocketmq/common/filter/impl/Type.java | 12 + .../alibaba/rocketmq/common/help/FAQUrl.java | 105 + .../rocketmq/common/hook/FilterCheckHook.java | 17 + .../rocketmq/common/message/Message.java | 239 + .../common/message/MessageAccessor.java | 71 + .../rocketmq/common/message/MessageConst.java | 78 + .../common/message/MessageDecoder.java | 284 + .../rocketmq/common/message/MessageExt.java | 275 + .../rocketmq/common/message/MessageId.java | 53 + .../rocketmq/common/message/MessageQueue.java | 137 + .../common/namesrv/NamesrvConfig.java | 43 + .../rocketmq/common/namesrv/NamesrvUtil.java | 10 + .../common/namesrv/RegisterBrokerResult.java | 44 + .../common/namesrv/TopAddressing.java | 90 + .../common/protocol/MQProtosHelper.java | 49 + .../rocketmq/common/protocol/RequestCode.java | 170 + .../common/protocol/ResponseCode.java | 60 + .../common/protocol/body/CMResult.java | 10 + .../common/protocol/body/ClusterInfo.java | 62 + .../common/protocol/body/Connection.java | 57 + .../common/protocol/body/ConsumeByWho.java | 83 + .../body/ConsumeMessageDirectlyResult.java | 70 + .../common/protocol/body/ConsumeStatus.java | 74 + .../protocol/body/ConsumerConnection.java | 88 + .../body/ConsumerOffsetSerializeWrapper.java | 27 + .../protocol/body/ConsumerRunningInfo.java | 323 + .../protocol/body/GetConsumerStatusBody.java | 41 + .../common/protocol/body/GroupList.java | 39 + .../common/protocol/body/KVTable.java | 39 + .../protocol/body/LockBatchRequestBody.java | 48 + .../protocol/body/LockBatchResponseBody.java | 28 + .../protocol/body/ProcessQueueInfo.java | 180 + .../protocol/body/ProducerConnection.java | 26 + .../body/QueryConsumeTimeSpanBody.java | 27 + .../body/QueryCorrectionOffsetBody.java | 40 + .../common/protocol/body/QueueTimeSpan.java | 75 + .../protocol/body/RegisterBrokerBody.java | 32 + .../common/protocol/body/ResetOffsetBody.java | 27 + .../body/SubscriptionGroupWrapper.java | 41 + .../body/TopicConfigSerializeWrapper.java | 34 + .../common/protocol/body/TopicList.java | 51 + .../protocol/body/UnlockBatchRequestBody.java | 48 + .../CheckTransactionStateRequestHeader.java | 44 + .../CheckTransactionStateResponseHeader.java | 80 + .../header/CloneGroupOffsetRequestHeader.java | 68 + ...umeMessageDirectlyResultRequestHeader.java | 63 + .../ConsumerSendMsgBackRequestHeader.java | 98 + .../header/CreateTopicRequestHeader.java | 127 + .../DeleteSubscriptionGroupRequestHeader.java | 32 + .../header/DeleteTopicRequestHeader.java | 32 + .../header/EndTransactionRequestHeader.java | 120 + .../header/EndTransactionResponseHeader.java | 21 + .../GetAllTopicConfigResponseHeader.java | 18 + .../header/GetBrokerConfigResponseHeader.java | 32 + .../header/GetConsumeStatsRequestHeader.java | 47 + ...etConsumerConnectionListRequestHeader.java | 34 + .../GetConsumerListByGroupRequestHeader.java | 29 + .../GetConsumerListByGroupResponseBody.java | 23 + .../GetConsumerListByGroupResponseHeader.java | 15 + .../GetConsumerRunningInfoRequestHeader.java | 54 + .../GetConsumerStatusRequestHeader.java | 58 + .../GetEarliestMsgStoretimeRequestHeader.java | 45 + ...GetEarliestMsgStoretimeResponseHeader.java | 32 + .../header/GetMaxOffsetRequestHeader.java | 45 + .../header/GetMaxOffsetResponseHeader.java | 32 + .../header/GetMinOffsetRequestHeader.java | 45 + .../header/GetMinOffsetResponseHeader.java | 32 + ...etProducerConnectionListRequestHeader.java | 34 + .../GetTopicStatsInfoRequestHeader.java | 30 + .../GetTopicsByClusterRequestHeader.java | 30 + ...NotifyConsumerIdsChangedRequestHeader.java | 30 + .../header/PullMessageRequestHeader.java | 141 + .../header/PullMessageResponseHeader.java | 68 + .../QueryConsumeTimeSpanRequestHeader.java | 44 + .../QueryConsumerOffsetRequestHeader.java | 56 + .../QueryConsumerOffsetResponseHeader.java | 34 + .../header/QueryCorrectionOffsetHeader.java | 59 + .../header/QueryMessageRequestHeader.java | 81 + .../header/QueryMessageResponseHeader.java | 44 + .../QueryTopicConsumeByWhoRequestHeader.java | 33 + .../header/ResetOffsetRequestHeader.java | 69 + .../header/SearchOffsetRequestHeader.java | 59 + .../header/SearchOffsetResponseHeader.java | 32 + .../header/SendMessageRequestHeader.java | 153 + .../header/SendMessageRequestHeaderV2.java | 188 + .../header/SendMessageResponseHeader.java | 56 + .../header/UnregisterClientRequestHeader.java | 60 + .../UnregisterClientResponseHeader.java | 26 + .../UpdateConsumerOffsetRequestHeader.java | 68 + .../UpdateConsumerOffsetResponseHeader.java | 18 + .../header/ViewMessageRequestHeader.java | 32 + .../header/ViewMessageResponseHeader.java | 20 + .../RegisterFilterServerRequestHeader.java | 26 + .../RegisterFilterServerResponseHeader.java | 38 + ...gisterMessageFilterClassRequestHeader.java | 62 + .../namesrv/DeleteKVConfigRequestHeader.java | 42 + .../DeleteTopicInNamesrvRequestHeader.java | 45 + .../namesrv/GetKVConfigRequestHeader.java | 42 + .../namesrv/GetKVConfigResponseHeader.java | 30 + .../GetKVListByNamespaceRequestHeader.java | 30 + .../namesrv/GetRouteInfoRequestHeader.java | 33 + .../namesrv/GetRouteInfoResponseHeader.java | 20 + .../namesrv/PutKVConfigRequestHeader.java | 50 + .../namesrv/RegisterBrokerRequestHeader.java | 80 + .../namesrv/RegisterBrokerResponseHeader.java | 42 + .../RegisterOrderTopicRequestHeader.java | 45 + .../UnRegisterBrokerRequestHeader.java | 68 + .../WipeWritePermOfBrokerRequestHeader.java | 47 + .../WipeWritePermOfBrokerResponseHeader.java | 45 + .../protocol/heartbeat/ConsumeType.java | 20 + .../protocol/heartbeat/ConsumerData.java | 90 + .../protocol/heartbeat/HeartbeatData.java | 56 + .../protocol/heartbeat/MessageModel.java | 24 + .../protocol/heartbeat/ProducerData.java | 27 + .../protocol/heartbeat/SubscriptionData.java | 163 + .../common/protocol/route/BrokerData.java | 100 + .../common/protocol/route/QueueData.java | 116 + .../common/protocol/route/TopicRouteData.java | 142 + .../protocol/topic/OffsetMovedEvent.java | 65 + .../rocketmq/common/running/RunningStats.java | 24 + .../rocketmq/common/stats/StatsItem.java | 277 + .../rocketmq/common/stats/StatsItemSet.java | 210 + .../rocketmq/common/stats/StatsSnapshot.java | 37 + .../subscription/SubscriptionGroupConfig.java | 167 + .../common/sysflag/MessageSysFlag.java | 51 + .../rocketmq/common/sysflag/PullSysFlag.java | 77 + .../common/sysflag/SubscriptionSysFlag.java | 58 + .../rocketmq/common/sysflag/TopicSysFlag.java | 80 + .../rocketmq/common/utils/HttpTinyClient.java | 150 + .../rocketmq/common/utils/IOTinyUtils.java | 165 + .../alibaba/rocketmq/common/MixAllTest.java | 23 + .../rocketmq/common/RemotingUtilTest.java | 29 + .../alibaba/rocketmq/common/UtilAllTest.java | 147 + .../rocketmq/common/filter/FilterAPITest.java | 20 + .../common/filter/PolishExprTest.java | 52 + .../common/protocol/ConsumeStatusTest.java | 20 + .../common/protocol/MQProtosHelperTest.java | 8 + rocketmq-example/pom.xml | 41 + .../com/alibaba/rocketmq/example/README.md | 7 + .../rocketmq/example/benchmark/Consumer.java | 192 + .../rocketmq/example/benchmark/Producer.java | 240 + .../benchmark/TransactionProducer.java | 278 + .../example/broadcast/PushConsumer.java | 63 + .../rocketmq/example/filter/Consumer.java | 54 + .../example/filter/MessageFilterImpl.java | 21 + .../rocketmq/example/filter/Producer.java | 48 + .../rocketmq/example/operation/Consumer.java | 122 + .../rocketmq/example/operation/Producer.java | 119 + .../example/ordermessage/Consumer.java | 77 + .../example/ordermessage/Producer.java | 77 + .../rocketmq/example/quickstart/Consumer.java | 58 + .../rocketmq/example/quickstart/Producer.java | 51 + .../rocketmq/example/simple/CachedQueue.java | 15 + .../rocketmq/example/simple/Producer.java | 68 + .../rocketmq/example/simple/PullConsumer.java | 86 + .../example/simple/PullConsumerTest.java | 51 + .../example/simple/PullScheduleService.java | 58 + .../rocketmq/example/simple/PushConsumer.java | 97 + .../example/simple/RandomAsyncCommit.java | 46 + .../rocketmq/example/simple/TestProducer.java | 75 + .../TransactionCheckListenerImpl.java | 49 + .../transaction/TransactionExecuterImpl.java | 48 + .../transaction/TransactionProducer.java | 67 + .../src/main/resources/MessageFilterImpl.java | 22 + rocketmq-filtersrv/pom.xml | 41 + .../com/alibaba/common/lang/ArrayUtil.java | 6413 +++++++++++++++++ .../lang/ClassInstantiationException.java | 58 + .../com/alibaba/common/lang/ClassUtil.java | 1367 ++++ .../lang/CloneNotSupportedException.java | 63 + .../alibaba/common/lang/ExceptionUtil.java | 31 + .../com/alibaba/common/lang/ObjectUtil.java | 457 ++ .../common/lang/ServiceNotFoundException.java | 117 + .../com/alibaba/common/lang/StringUtil.java | 4949 +++++++++++++ .../common/lang/exception/ChainedError.java | 113 + .../lang/exception/ChainedException.java | 113 + .../exception/ChainedRuntimeException.java | 113 + .../lang/exception/ChainedThrowable.java | 54 + .../exception/ChainedThrowableDelegate.java | 252 + .../lang/exception/ExceptionHelper.java | 418 ++ .../filtersrv/FilterServerOuterAPI.java | 84 + .../rocketmq/filtersrv/FiltersrvConfig.java | 144 + .../filtersrv/FiltersrvController.java | 242 + .../rocketmq/filtersrv/FiltersrvStartup.java | 203 + .../rocketmq/filtersrv/filter/DynaCode.java | 496 ++ .../filter/FilterClassFetchMethod.java | 5 + .../filtersrv/filter/FilterClassInfo.java | 40 + .../filtersrv/filter/FilterClassLoader.java | 7 + .../filtersrv/filter/FilterClassManager.java | 168 + .../filter/HttpFilterClassFetchMethod.java | 38 + .../processor/DefaultRequestProcessor.java | 379 + .../stats/FilterServerStatsManager.java | 49 + rocketmq-namesrv/pom.xml | 33 + .../rocketmq/namesrv/NamesrvController.java | 166 + .../rocketmq/namesrv/NamesrvStartup.java | 185 + .../namesrv/kvconfig/KVConfigManager.java | 283 + .../kvconfig/KVConfigSerializeWrapper.java | 41 + .../processor/DefaultRequestProcessor.java | 590 ++ .../routeinfo/BrokerHousekeepingService.java | 63 + .../namesrv/routeinfo/RouteInfoManager.java | 939 +++ rocketmq-remoting/pom.xml | 34 + .../remoting/ChannelEventListener.java | 38 + .../remoting/CommandCustomHeader.java | 29 + .../rocketmq/remoting/InvokeCallback.java | 29 + .../alibaba/rocketmq/remoting/RPCHook.java | 11 + .../rocketmq/remoting/RemotingClient.java | 63 + .../rocketmq/remoting/RemotingServer.java | 73 + .../rocketmq/remoting/RemotingService.java | 11 + .../remoting/annotation/CFNotNull.java | 35 + .../remoting/annotation/CFNullable.java | 35 + .../rocketmq/remoting/common/Pair.java | 53 + .../remoting/common/RemotingHelper.java | 236 + .../remoting/common/RemotingUtil.java | 237 + .../common/SemaphoreReleaseOnlyOnce.java | 50 + .../remoting/common/ServiceThread.java | 156 + .../exception/RemotingCommandException.java | 36 + .../exception/RemotingConnectException.java | 36 + .../remoting/exception/RemotingException.java | 36 + .../RemotingSendRequestException.java | 36 + .../exception/RemotingTimeoutException.java | 42 + .../RemotingTooMuchRequestException.java | 31 + .../remoting/netty/NettyClientConfig.java | 139 + .../rocketmq/remoting/netty/NettyDecoder.java | 75 + .../rocketmq/remoting/netty/NettyEncoder.java | 62 + .../rocketmq/remoting/netty/NettyEvent.java | 59 + .../remoting/netty/NettyEventType.java | 29 + .../remoting/netty/NettyRemotingAbstract.java | 510 ++ .../remoting/netty/NettyRemotingClient.java | 776 ++ .../remoting/netty/NettyRemotingServer.java | 410 ++ .../remoting/netty/NettyRequestProcessor.java | 32 + .../remoting/netty/NettyServerConfig.java | 136 + .../remoting/netty/NettySystemConfig.java | 29 + .../remoting/netty/ResponseFuture.java | 150 + .../remoting/protocol/LanguageCode.java | 12 + .../remoting/protocol/RemotingCommand.java | 506 ++ .../protocol/RemotingCommandType.java | 27 + .../protocol/RemotingSerializable.java | 72 + .../protocol/RemotingSysResponseCode.java | 12 + .../rocketmq/remoting/protocol/protocol.txt | 12 + .../rocketmq/remoting/version/V3_1_9.java | 5 + .../rocketmq/remoting/ExceptionTest.java | 83 + .../alibaba/rocketmq/remoting/MixTest.java | 17 + .../remoting/NettyConnectionTest.java | 47 + .../rocketmq/remoting/NettyIdleTest.java | 75 + .../rocketmq/remoting/NettyRPCTest.java | 240 + .../rocketmq/remoting/SyncInvokeTest.java | 39 + .../rocketmq/subclass/TestSubClassAuto.java | 17 + rocketmq-srvutil/pom.xml | 38 + .../alibaba/rocketmq/srvutil/ServerUtil.java | 95 + .../rocketmq/srvutil/ServerUtilTest.java | 11 + rocketmq-store/pom.xml | 25 + rocketmq-store/sbin/map.sh | 1 + rocketmq-store/sbin/put.sh | 1 + rocketmq-store/sbin/response.sh | 1 + rocketmq-store/sbin/showcpu.sh | 6 + rocketmq-store/sbin/showdirty.sh | 9 + rocketmq-store/sbin/showdirty2.sh | 9 + rocketmq-store/sbin/showiostat.sh | 2 + rocketmq-store/sbin/showload.sh | 6 + rocketmq-store/sbin/showmap.sh | 2 + rocketmq-store/sbin/showmaptotal.sh | 6 + rocketmq-store/sbin/showvmstat.sh | 8 + rocketmq-store/sbin/test.sh | 6 + .../store/AllocateMapedFileService.java | 236 + .../rocketmq/store/AppendMessageCallback.java | 42 + .../rocketmq/store/AppendMessageResult.java | 127 + .../rocketmq/store/AppendMessageStatus.java | 33 + .../com/alibaba/rocketmq/store/CommitLog.java | 1179 +++ .../alibaba/rocketmq/store/ConsumeQueue.java | 577 ++ .../rocketmq/store/DefaultMessageFilter.java | 45 + .../rocketmq/store/DefaultMessageStore.java | 1930 +++++ .../rocketmq/store/DispatchRequest.java | 146 + .../rocketmq/store/GetMessageResult.java | 149 + .../rocketmq/store/GetMessageStatus.java | 43 + .../com/alibaba/rocketmq/store/MapedFile.java | 466 ++ .../rocketmq/store/MapedFileQueue.java | 626 ++ .../rocketmq/store/MessageExtBrokerInner.java | 63 + .../alibaba/rocketmq/store/MessageFilter.java | 29 + .../alibaba/rocketmq/store/MessageStore.java | 205 + .../rocketmq/store/PutMessageResult.java | 66 + .../rocketmq/store/PutMessageStatus.java | 33 + .../rocketmq/store/QueryMessageResult.java | 83 + .../rocketmq/store/ReferenceResource.java | 107 + .../alibaba/rocketmq/store/RunningFlags.java | 147 + .../store/SelectMapedBufferResult.java | 89 + .../rocketmq/store/StoreCheckpoint.java | 146 + .../rocketmq/store/StoreStatsService.java | 610 ++ .../com/alibaba/rocketmq/store/StoreUtil.java | 40 + .../rocketmq/store/config/BrokerRole.java | 31 + .../rocketmq/store/config/FlushDiskType.java | 33 + .../store/config/MessageStoreConfig.java | 595 ++ .../store/config/StorePathConfigHelper.java | 57 + .../rocketmq/store/ha/HAConnection.java | 473 ++ .../alibaba/rocketmq/store/ha/HAService.java | 693 ++ .../rocketmq/store/ha/WaitNotifyObject.java | 115 + .../rocketmq/store/index/IndexFile.java | 322 + .../rocketmq/store/index/IndexHeader.java | 144 + .../rocketmq/store/index/IndexService.java | 443 ++ .../store/index/QueryOffsetResult.java | 54 + .../schedule/DelayOffsetSerializeWrapper.java | 42 + .../schedule/ScheduleMessageService.java | 411 ++ .../rocketmq/store/stats/BrokerStats.java | 117 + .../store/stats/BrokerStatsManager.java | 148 + .../store/DefaultMessageStoreTest.java | 173 + .../rocketmq/store/MapedFileQueueTest.java | 206 + .../alibaba/rocketmq/store/MapedFileTest.java | 94 + .../alibaba/rocketmq/store/RecoverTest.java | 253 + .../rocketmq/store/StoreCheckpointTest.java | 51 + .../rocketmq/store/index/IndexFileTest.java | 77 + .../store/schedule/ScheduleMessageTest.java | 124 + rocketmq-tools/pom.xml | 45 + .../tools/admin/DefaultMQAdminExt.java | 436 ++ .../tools/admin/DefaultMQAdminExtImpl.java | 840 +++ .../rocketmq/tools/admin/MQAdminExt.java | 611 ++ .../tools/admin/api/MessageTrack.java | 44 + .../rocketmq/tools/admin/api/TrackType.java | 14 + .../rocketmq/tools/command/CommandUtil.java | 98 + .../tools/command/MQAdminStartup.java | 227 + .../rocketmq/tools/command/SubCommand.java | 41 + .../broker/BrokerStatusSubCommand.java | 92 + .../broker/CleanExpiredCQSubCommand.java | 71 + .../broker/UpdateBrokerConfigSubCommand.java | 119 + .../cluster/ClusterListSubCommand.java | 256 + .../ConsumerConnectionSubCommand.java | 116 + .../ProducerConnectionSubCommand.java | 97 + .../consumer/ConsumerProgressSubCommand.java | 321 + .../consumer/ConsumerStatusSubCommand.java | 166 + .../DeleteSubscriptionGroupCommand.java | 121 + .../consumer/StartMonitoringSubCommand.java | 72 + .../consumer/UpdateSubGroupSubCommand.java | 188 + .../command/message/CheckMsgSubCommand.java | 58 + .../message/PrintMessageSubCommand.java | 173 + .../message/QueryMsgByIdSubCommand.java | 236 + .../message/QueryMsgByKeySubCommand.java | 98 + .../message/QueryMsgByOffsetSubCommand.java | 118 + .../rocketmq/tools/command/message/Store.java | 263 + .../namesrv/DeleteKvConfigCommand.java | 81 + .../namesrv/DeleteProjectGroupCommand.java | 91 + .../namesrv/GetProjectGroupCommand.java | 94 + .../namesrv/UpdateKvConfigCommand.java | 87 + .../namesrv/UpdateProjectGroupCommand.java | 81 + .../namesrv/WipeWritePermSubCommand.java | 96 + .../offset/CloneGroupOffsetCommand.java | 100 + .../offset/GetConsumerStatusCommand.java | 115 + .../offset/ResetOffsetByTimeCommand.java | 140 + .../offset/ResetOffsetByTimeOldCommand.java | 126 + .../command/topic/DeleteTopicSubCommand.java | 115 + .../command/topic/TopicListSubCommand.java | 155 + .../command/topic/TopicRouteSubCommand.java | 79 + .../command/topic/TopicStatusSubCommand.java | 113 + .../command/topic/UpdateOrderConfCommand.java | 120 + .../command/topic/UpdateTopicSubCommand.java | 201 + .../tools/monitor/DefaultMonitorListener.java | 83 + .../tools/monitor/DeleteMsgsEvent.java | 36 + .../rocketmq/tools/monitor/FailedMsgs.java | 44 + .../rocketmq/tools/monitor/MonitorConfig.java | 31 + .../tools/monitor/MonitorListener.java | 46 + .../tools/monitor/MonitorService.java | 332 + .../rocketmq/tools/monitor/UndoneMsgs.java | 70 + test/consumer.sh | 6 + test/producer.sh | 6 + test/runclass.sh | 31 + 577 files changed, 86096 insertions(+), 4 deletions(-) create mode 100644 .gitignore create mode 100644 .travis.yml create mode 100644 LICENSE.txt create mode 100644 benchmark/consumer.sh create mode 100644 benchmark/producer.sh create mode 100644 benchmark/runclass.sh create mode 100644 benchmark/tproducer.sh create mode 100644 bin/README.md create mode 100644 bin/mqadmin create mode 100644 bin/mqadmin.xml create mode 100644 bin/mqbroker create mode 100644 bin/mqbroker.xml create mode 100644 bin/mqfiltersrv create mode 100644 bin/mqfiltersrv.xml create mode 100644 bin/mqnamesrv create mode 100644 bin/mqnamesrv.xml create mode 100644 bin/mqshutdown create mode 100644 bin/os.sh create mode 100644 bin/play.sh create mode 100644 bin/runbroker.sh create mode 100644 bin/runserver.sh create mode 100644 bin/startfsrv.sh create mode 100644 bin/tools.sh create mode 100644 conf/2m-2s-async/broker-a-s.properties create mode 100644 conf/2m-2s-async/broker-a.properties create mode 100644 conf/2m-2s-async/broker-b-s.properties create mode 100644 conf/2m-2s-async/broker-b.properties create mode 100644 conf/2m-2s-sync/broker-a-s.properties create mode 100644 conf/2m-2s-sync/broker-a.properties create mode 100644 conf/2m-2s-sync/broker-b-s.properties create mode 100644 conf/2m-2s-sync/broker-b.properties create mode 100644 conf/2m-noslave/broker-a.properties create mode 100644 conf/2m-noslave/broker-b.properties create mode 100644 conf/logback_broker.xml create mode 100644 conf/logback_filtersrv.xml create mode 100644 conf/logback_namesrv.xml create mode 100644 conf/logback_tools.xml create mode 100644 deploy.bat create mode 100644 develop.sh create mode 100644 docs/rocketmq.java.code.style.xml create mode 100644 eclipse.bat create mode 100644 install.bat create mode 100644 install.sh create mode 100644 pom.xml create mode 100644 release-client.xml create mode 100644 release.xml create mode 100644 rocketmq-broker/pom.xml create mode 100644 rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/BrokerController.java create mode 100644 rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/BrokerPathConfigHelper.java create mode 100644 rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/BrokerStartup.java create mode 100644 rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/client/ClientChannelInfo.java create mode 100644 rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/client/ClientHousekeepingService.java create mode 100644 rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/client/ConsumerGroupInfo.java create mode 100644 rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/client/ConsumerIdsChangeListener.java create mode 100644 rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/client/ConsumerManager.java create mode 100644 rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/client/DefaultConsumerIdsChangeListener.java create mode 100644 rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/client/ProducerManager.java create mode 100644 rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/client/net/Broker2Client.java create mode 100644 rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/client/rebalance/RebalanceLockManager.java create mode 100644 rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/filtersrv/FilterServerManager.java create mode 100644 rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/filtersrv/FilterServerUtil.java create mode 100644 rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/longpolling/ManyPullRequest.java create mode 100644 rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/longpolling/PullRequest.java create mode 100644 rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/longpolling/PullRequestHoldService.java create mode 100644 rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/mqtrace/ConsumeMessageContext.java create mode 100644 rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/mqtrace/ConsumeMessageHook.java create mode 100644 rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/mqtrace/SendMessageContext.java create mode 100644 rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/mqtrace/SendMessageHook.java create mode 100644 rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/offset/ConsumerOffsetManager.java create mode 100644 rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/out/BrokerOuterAPI.java create mode 100644 rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/pagecache/ManyMessageTransfer.java create mode 100644 rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/pagecache/OneMessageTransfer.java create mode 100644 rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/pagecache/QueryMessageTransfer.java create mode 100644 rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/processor/AdminBrokerProcessor.java create mode 100644 rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/processor/ClientManageProcessor.java create mode 100644 rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/processor/EndTransactionProcessor.java create mode 100644 rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/processor/ForwardRequestProcessor.java create mode 100644 rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/processor/PullMessageProcessor.java create mode 100644 rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/processor/QueryMessageProcessor.java create mode 100644 rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/processor/SendMessageProcessor.java create mode 100644 rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/slave/SlaveSynchronize.java create mode 100644 rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/subscription/SubscriptionGroupManager.java create mode 100644 rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/topic/TopicConfigManager.java create mode 100644 rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/transaction/TransactionRecord.java create mode 100644 rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/transaction/TransactionStore.java create mode 100644 rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/transaction/jdbc/JDBCTransactionStore.java create mode 100644 rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/transaction/jdbc/JDBCTransactionStoreConfig.java create mode 100644 rocketmq-broker/src/main/resources/transaction.sql create mode 100644 rocketmq-broker/src/test/java/com/alibaba/rocketmq/broker/api/SendMessageTest.java create mode 100644 rocketmq-broker/src/test/java/com/alibaba/rocketmq/broker/offset/ConsumerOffsetManagerTest.java create mode 100644 rocketmq-broker/src/test/java/com/alibaba/rocketmq/broker/topic/TopicConfigManagerTest.java create mode 100644 rocketmq-broker/src/test/java/com/alibaba/rocketmq/broker/transaction/jdbc/JDBCTransactionStoreTest.java create mode 100644 rocketmq-client/pom.xml create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/ClientConfig.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/MQAdmin.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/MQHelper.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/QueryResult.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/Validators.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/VirtualEnvUtil.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/admin/MQAdminExtInner.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/AllocateMessageQueueStrategy.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/DefaultMQPullConsumer.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/DefaultMQPushConsumer.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/MQConsumer.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/MQPullConsumer.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/MQPullConsumerScheduleService.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/MQPushConsumer.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/MessageQueueListener.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/PullCallback.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/PullResult.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/PullStatus.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/PullTaskCallback.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/PullTaskContext.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/listener/ConsumeConcurrentlyContext.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/listener/ConsumeConcurrentlyStatus.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/listener/ConsumeOrderlyContext.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/listener/ConsumeOrderlyStatus.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/listener/MessageListener.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/listener/MessageListenerConcurrently.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/listener/MessageListenerOrderly.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/rebalance/AllocateMessageQueueAveragely.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/rebalance/AllocateMessageQueueAveragelyByCircle.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/rebalance/AllocateMessageQueueByConfig.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/rebalance/AllocateMessageQueueByMachineRoom.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/store/LocalFileOffsetStore.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/store/OffsetSerializeWrapper.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/store/OffsetStore.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/store/ReadOffsetType.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/store/RemoteBrokerOffsetStore.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/exception/MQBrokerException.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/exception/MQClientException.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/hook/CheckForbiddenContext.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/hook/CheckForbiddenHook.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/hook/ConsumeMessageContext.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/hook/ConsumeMessageHook.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/hook/FilterMessageContext.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/hook/FilterMessageHook.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/hook/SendMessageContext.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/hook/SendMessageHook.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/ClientRemotingProcessor.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/CommunicationMode.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/FindBrokerResult.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/MQAdminImpl.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/MQClientAPIImpl.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/MQClientManager.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/ConsumeMessageConcurrentlyService.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/ConsumeMessageOrderlyService.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/ConsumeMessageService.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/MQConsumerInner.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/MessageQueueLock.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/ProcessQueue.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/PullAPIWrapper.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/PullMessageService.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/PullRequest.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/PullResultExt.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/RebalanceImpl.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/RebalancePullImpl.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/RebalancePushImpl.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/RebalanceService.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/factory/MQClientInstance.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/producer/DefaultMQProducerImpl.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/producer/MQProducerInner.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/producer/TopicPublishInfo.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/log/ClientLogger.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/DefaultMQProducer.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/LocalTransactionExecuter.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/LocalTransactionState.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/MQProducer.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/MessageQueueSelector.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/SendCallback.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/SendResult.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/SendStatus.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/TransactionCheckListener.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/TransactionMQProducer.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/TransactionSendResult.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/selector/SelectMessageQueueByHash.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/selector/SelectMessageQueueByMachineRoom.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/selector/SelectMessageQueueByRandoom.java create mode 100644 rocketmq-client/src/main/java/com/alibaba/rocketmq/client/stat/ConsumerStatsManager.java create mode 100644 rocketmq-client/src/main/resources/log4j_rocketmq_client.xml create mode 100644 rocketmq-client/src/main/resources/logback_rocketmq_client.xml create mode 100644 rocketmq-client/src/test/java/com/alibaba/rocketmq/client/TestMultiConsumerProducer.java create mode 100644 rocketmq-client/src/test/java/com/alibaba/rocketmq/client/ValidatorsTest.java create mode 100644 rocketmq-client/src/test/java/com/alibaba/rocketmq/client/consumer/loadbalance/AllocateMessageQueueAveragelyTest.java create mode 100644 rocketmq-common/pom.xml create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/BrokerConfig.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/ConfigManager.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/DataVersion.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/MQVersion.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/MixAll.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/Pair.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/ServiceState.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/ServiceThread.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/SystemClock.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/ThreadFactoryImpl.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/TopicConfig.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/TopicFilterType.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/UtilAll.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/admin/ConsumeStats.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/admin/OffsetWrapper.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/admin/RollbackStats.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/admin/TopicOffset.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/admin/TopicStatsTable.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/annotation/ImportantField.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/constant/LoggerName.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/constant/PermName.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/consumer/ConsumeFromWhere.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/filter/FilterAPI.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/filter/MessageFilter.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/filter/impl/Op.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/filter/impl/Operand.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/filter/impl/Operator.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/filter/impl/PolishExpr.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/filter/impl/Type.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/help/FAQUrl.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/hook/FilterCheckHook.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/message/Message.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/message/MessageAccessor.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/message/MessageConst.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/message/MessageDecoder.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/message/MessageExt.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/message/MessageId.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/message/MessageQueue.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/namesrv/NamesrvConfig.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/namesrv/NamesrvUtil.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/namesrv/RegisterBrokerResult.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/namesrv/TopAddressing.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/MQProtosHelper.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/RequestCode.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/ResponseCode.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/CMResult.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ClusterInfo.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/Connection.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ConsumeByWho.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ConsumeMessageDirectlyResult.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ConsumeStatus.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ConsumerConnection.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ConsumerOffsetSerializeWrapper.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ConsumerRunningInfo.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/GetConsumerStatusBody.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/GroupList.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/KVTable.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/LockBatchRequestBody.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/LockBatchResponseBody.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ProcessQueueInfo.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ProducerConnection.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/QueryConsumeTimeSpanBody.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/QueryCorrectionOffsetBody.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/QueueTimeSpan.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/RegisterBrokerBody.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ResetOffsetBody.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/SubscriptionGroupWrapper.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/TopicConfigSerializeWrapper.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/TopicList.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/UnlockBatchRequestBody.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/CheckTransactionStateRequestHeader.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/CheckTransactionStateResponseHeader.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/CloneGroupOffsetRequestHeader.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/ConsumeMessageDirectlyResultRequestHeader.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/ConsumerSendMsgBackRequestHeader.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/CreateTopicRequestHeader.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/DeleteSubscriptionGroupRequestHeader.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/DeleteTopicRequestHeader.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/EndTransactionRequestHeader.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/EndTransactionResponseHeader.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetAllTopicConfigResponseHeader.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetBrokerConfigResponseHeader.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetConsumeStatsRequestHeader.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetConsumerConnectionListRequestHeader.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetConsumerListByGroupRequestHeader.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetConsumerListByGroupResponseBody.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetConsumerListByGroupResponseHeader.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetConsumerRunningInfoRequestHeader.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetConsumerStatusRequestHeader.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetEarliestMsgStoretimeRequestHeader.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetEarliestMsgStoretimeResponseHeader.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetMaxOffsetRequestHeader.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetMaxOffsetResponseHeader.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetMinOffsetRequestHeader.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetMinOffsetResponseHeader.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetProducerConnectionListRequestHeader.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetTopicStatsInfoRequestHeader.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetTopicsByClusterRequestHeader.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/NotifyConsumerIdsChangedRequestHeader.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/PullMessageRequestHeader.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/PullMessageResponseHeader.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/QueryConsumeTimeSpanRequestHeader.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/QueryConsumerOffsetRequestHeader.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/QueryConsumerOffsetResponseHeader.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/QueryCorrectionOffsetHeader.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/QueryMessageRequestHeader.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/QueryMessageResponseHeader.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/QueryTopicConsumeByWhoRequestHeader.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/ResetOffsetRequestHeader.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/SearchOffsetRequestHeader.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/SearchOffsetResponseHeader.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/SendMessageRequestHeader.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/SendMessageRequestHeaderV2.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/SendMessageResponseHeader.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/UnregisterClientRequestHeader.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/UnregisterClientResponseHeader.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/UpdateConsumerOffsetRequestHeader.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/UpdateConsumerOffsetResponseHeader.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/ViewMessageRequestHeader.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/ViewMessageResponseHeader.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/filtersrv/RegisterFilterServerRequestHeader.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/filtersrv/RegisterFilterServerResponseHeader.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/filtersrv/RegisterMessageFilterClassRequestHeader.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/DeleteKVConfigRequestHeader.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/DeleteTopicInNamesrvRequestHeader.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/GetKVConfigRequestHeader.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/GetKVConfigResponseHeader.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/GetKVListByNamespaceRequestHeader.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/GetRouteInfoRequestHeader.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/GetRouteInfoResponseHeader.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/PutKVConfigRequestHeader.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/RegisterBrokerRequestHeader.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/RegisterBrokerResponseHeader.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/RegisterOrderTopicRequestHeader.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/UnRegisterBrokerRequestHeader.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/WipeWritePermOfBrokerRequestHeader.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/WipeWritePermOfBrokerResponseHeader.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/heartbeat/ConsumeType.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/heartbeat/ConsumerData.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/heartbeat/HeartbeatData.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/heartbeat/MessageModel.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/heartbeat/ProducerData.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/heartbeat/SubscriptionData.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/route/BrokerData.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/route/QueueData.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/route/TopicRouteData.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/topic/OffsetMovedEvent.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/running/RunningStats.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/stats/StatsItem.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/stats/StatsItemSet.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/stats/StatsSnapshot.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/subscription/SubscriptionGroupConfig.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/sysflag/MessageSysFlag.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/sysflag/PullSysFlag.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/sysflag/SubscriptionSysFlag.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/sysflag/TopicSysFlag.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/utils/HttpTinyClient.java create mode 100644 rocketmq-common/src/main/java/com/alibaba/rocketmq/common/utils/IOTinyUtils.java create mode 100644 rocketmq-common/src/test/java/com/alibaba/rocketmq/common/MixAllTest.java create mode 100644 rocketmq-common/src/test/java/com/alibaba/rocketmq/common/RemotingUtilTest.java create mode 100644 rocketmq-common/src/test/java/com/alibaba/rocketmq/common/UtilAllTest.java create mode 100644 rocketmq-common/src/test/java/com/alibaba/rocketmq/common/filter/FilterAPITest.java create mode 100644 rocketmq-common/src/test/java/com/alibaba/rocketmq/common/filter/PolishExprTest.java create mode 100644 rocketmq-common/src/test/java/com/alibaba/rocketmq/common/protocol/ConsumeStatusTest.java create mode 100644 rocketmq-common/src/test/java/com/alibaba/rocketmq/common/protocol/MQProtosHelperTest.java create mode 100644 rocketmq-example/pom.xml create mode 100644 rocketmq-example/src/main/java/com/alibaba/rocketmq/example/README.md create mode 100644 rocketmq-example/src/main/java/com/alibaba/rocketmq/example/benchmark/Consumer.java create mode 100644 rocketmq-example/src/main/java/com/alibaba/rocketmq/example/benchmark/Producer.java create mode 100644 rocketmq-example/src/main/java/com/alibaba/rocketmq/example/benchmark/TransactionProducer.java create mode 100644 rocketmq-example/src/main/java/com/alibaba/rocketmq/example/broadcast/PushConsumer.java create mode 100644 rocketmq-example/src/main/java/com/alibaba/rocketmq/example/filter/Consumer.java create mode 100644 rocketmq-example/src/main/java/com/alibaba/rocketmq/example/filter/MessageFilterImpl.java create mode 100644 rocketmq-example/src/main/java/com/alibaba/rocketmq/example/filter/Producer.java create mode 100644 rocketmq-example/src/main/java/com/alibaba/rocketmq/example/operation/Consumer.java create mode 100644 rocketmq-example/src/main/java/com/alibaba/rocketmq/example/operation/Producer.java create mode 100644 rocketmq-example/src/main/java/com/alibaba/rocketmq/example/ordermessage/Consumer.java create mode 100644 rocketmq-example/src/main/java/com/alibaba/rocketmq/example/ordermessage/Producer.java create mode 100644 rocketmq-example/src/main/java/com/alibaba/rocketmq/example/quickstart/Consumer.java create mode 100644 rocketmq-example/src/main/java/com/alibaba/rocketmq/example/quickstart/Producer.java create mode 100644 rocketmq-example/src/main/java/com/alibaba/rocketmq/example/simple/CachedQueue.java create mode 100644 rocketmq-example/src/main/java/com/alibaba/rocketmq/example/simple/Producer.java create mode 100644 rocketmq-example/src/main/java/com/alibaba/rocketmq/example/simple/PullConsumer.java create mode 100644 rocketmq-example/src/main/java/com/alibaba/rocketmq/example/simple/PullConsumerTest.java create mode 100644 rocketmq-example/src/main/java/com/alibaba/rocketmq/example/simple/PullScheduleService.java create mode 100644 rocketmq-example/src/main/java/com/alibaba/rocketmq/example/simple/PushConsumer.java create mode 100644 rocketmq-example/src/main/java/com/alibaba/rocketmq/example/simple/RandomAsyncCommit.java create mode 100644 rocketmq-example/src/main/java/com/alibaba/rocketmq/example/simple/TestProducer.java create mode 100644 rocketmq-example/src/main/java/com/alibaba/rocketmq/example/transaction/TransactionCheckListenerImpl.java create mode 100644 rocketmq-example/src/main/java/com/alibaba/rocketmq/example/transaction/TransactionExecuterImpl.java create mode 100644 rocketmq-example/src/main/java/com/alibaba/rocketmq/example/transaction/TransactionProducer.java create mode 100644 rocketmq-example/src/main/resources/MessageFilterImpl.java create mode 100644 rocketmq-filtersrv/pom.xml create mode 100644 rocketmq-filtersrv/src/main/java/com/alibaba/common/lang/ArrayUtil.java create mode 100644 rocketmq-filtersrv/src/main/java/com/alibaba/common/lang/ClassInstantiationException.java create mode 100644 rocketmq-filtersrv/src/main/java/com/alibaba/common/lang/ClassUtil.java create mode 100644 rocketmq-filtersrv/src/main/java/com/alibaba/common/lang/CloneNotSupportedException.java create mode 100644 rocketmq-filtersrv/src/main/java/com/alibaba/common/lang/ExceptionUtil.java create mode 100644 rocketmq-filtersrv/src/main/java/com/alibaba/common/lang/ObjectUtil.java create mode 100644 rocketmq-filtersrv/src/main/java/com/alibaba/common/lang/ServiceNotFoundException.java create mode 100644 rocketmq-filtersrv/src/main/java/com/alibaba/common/lang/StringUtil.java create mode 100644 rocketmq-filtersrv/src/main/java/com/alibaba/common/lang/exception/ChainedError.java create mode 100644 rocketmq-filtersrv/src/main/java/com/alibaba/common/lang/exception/ChainedException.java create mode 100644 rocketmq-filtersrv/src/main/java/com/alibaba/common/lang/exception/ChainedRuntimeException.java create mode 100644 rocketmq-filtersrv/src/main/java/com/alibaba/common/lang/exception/ChainedThrowable.java create mode 100644 rocketmq-filtersrv/src/main/java/com/alibaba/common/lang/exception/ChainedThrowableDelegate.java create mode 100644 rocketmq-filtersrv/src/main/java/com/alibaba/common/lang/exception/ExceptionHelper.java create mode 100644 rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/FilterServerOuterAPI.java create mode 100644 rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/FiltersrvConfig.java create mode 100644 rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/FiltersrvController.java create mode 100644 rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/FiltersrvStartup.java create mode 100644 rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/filter/DynaCode.java create mode 100644 rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/filter/FilterClassFetchMethod.java create mode 100644 rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/filter/FilterClassInfo.java create mode 100644 rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/filter/FilterClassLoader.java create mode 100644 rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/filter/FilterClassManager.java create mode 100644 rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/filter/HttpFilterClassFetchMethod.java create mode 100644 rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/processor/DefaultRequestProcessor.java create mode 100644 rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/stats/FilterServerStatsManager.java create mode 100644 rocketmq-namesrv/pom.xml create mode 100644 rocketmq-namesrv/src/main/java/com/alibaba/rocketmq/namesrv/NamesrvController.java create mode 100644 rocketmq-namesrv/src/main/java/com/alibaba/rocketmq/namesrv/NamesrvStartup.java create mode 100644 rocketmq-namesrv/src/main/java/com/alibaba/rocketmq/namesrv/kvconfig/KVConfigManager.java create mode 100644 rocketmq-namesrv/src/main/java/com/alibaba/rocketmq/namesrv/kvconfig/KVConfigSerializeWrapper.java create mode 100644 rocketmq-namesrv/src/main/java/com/alibaba/rocketmq/namesrv/processor/DefaultRequestProcessor.java create mode 100644 rocketmq-namesrv/src/main/java/com/alibaba/rocketmq/namesrv/routeinfo/BrokerHousekeepingService.java create mode 100644 rocketmq-namesrv/src/main/java/com/alibaba/rocketmq/namesrv/routeinfo/RouteInfoManager.java create mode 100644 rocketmq-remoting/pom.xml create mode 100644 rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/ChannelEventListener.java create mode 100644 rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/CommandCustomHeader.java create mode 100644 rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/InvokeCallback.java create mode 100644 rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/RPCHook.java create mode 100644 rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/RemotingClient.java create mode 100644 rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/RemotingServer.java create mode 100644 rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/RemotingService.java create mode 100644 rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/annotation/CFNotNull.java create mode 100644 rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/annotation/CFNullable.java create mode 100644 rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/common/Pair.java create mode 100644 rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/common/RemotingHelper.java create mode 100644 rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/common/RemotingUtil.java create mode 100644 rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/common/SemaphoreReleaseOnlyOnce.java create mode 100644 rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/common/ServiceThread.java create mode 100644 rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/exception/RemotingCommandException.java create mode 100644 rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/exception/RemotingConnectException.java create mode 100644 rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/exception/RemotingException.java create mode 100644 rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/exception/RemotingSendRequestException.java create mode 100644 rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/exception/RemotingTimeoutException.java create mode 100644 rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/exception/RemotingTooMuchRequestException.java create mode 100644 rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettyClientConfig.java create mode 100644 rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettyDecoder.java create mode 100644 rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettyEncoder.java create mode 100644 rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettyEvent.java create mode 100644 rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettyEventType.java create mode 100644 rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettyRemotingAbstract.java create mode 100644 rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettyRemotingClient.java create mode 100644 rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettyRemotingServer.java create mode 100644 rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettyRequestProcessor.java create mode 100644 rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettyServerConfig.java create mode 100644 rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettySystemConfig.java create mode 100644 rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/ResponseFuture.java create mode 100644 rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/protocol/LanguageCode.java create mode 100644 rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/protocol/RemotingCommand.java create mode 100644 rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/protocol/RemotingCommandType.java create mode 100644 rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/protocol/RemotingSerializable.java create mode 100644 rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/protocol/RemotingSysResponseCode.java create mode 100644 rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/protocol/protocol.txt create mode 100644 rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/version/V3_1_9.java create mode 100644 rocketmq-remoting/src/test/java/com/alibaba/rocketmq/remoting/ExceptionTest.java create mode 100644 rocketmq-remoting/src/test/java/com/alibaba/rocketmq/remoting/MixTest.java create mode 100644 rocketmq-remoting/src/test/java/com/alibaba/rocketmq/remoting/NettyConnectionTest.java create mode 100644 rocketmq-remoting/src/test/java/com/alibaba/rocketmq/remoting/NettyIdleTest.java create mode 100644 rocketmq-remoting/src/test/java/com/alibaba/rocketmq/remoting/NettyRPCTest.java create mode 100644 rocketmq-remoting/src/test/java/com/alibaba/rocketmq/remoting/SyncInvokeTest.java create mode 100644 rocketmq-remoting/src/test/java/com/alibaba/rocketmq/subclass/TestSubClassAuto.java create mode 100644 rocketmq-srvutil/pom.xml create mode 100644 rocketmq-srvutil/src/main/java/com/alibaba/rocketmq/srvutil/ServerUtil.java create mode 100644 rocketmq-srvutil/src/test/java/com/alibaba/rocketmq/srvutil/ServerUtilTest.java create mode 100644 rocketmq-store/pom.xml create mode 100644 rocketmq-store/sbin/map.sh create mode 100644 rocketmq-store/sbin/put.sh create mode 100644 rocketmq-store/sbin/response.sh create mode 100644 rocketmq-store/sbin/showcpu.sh create mode 100644 rocketmq-store/sbin/showdirty.sh create mode 100644 rocketmq-store/sbin/showdirty2.sh create mode 100644 rocketmq-store/sbin/showiostat.sh create mode 100644 rocketmq-store/sbin/showload.sh create mode 100644 rocketmq-store/sbin/showmap.sh create mode 100644 rocketmq-store/sbin/showmaptotal.sh create mode 100644 rocketmq-store/sbin/showvmstat.sh create mode 100644 rocketmq-store/sbin/test.sh create mode 100644 rocketmq-store/src/main/java/com/alibaba/rocketmq/store/AllocateMapedFileService.java create mode 100644 rocketmq-store/src/main/java/com/alibaba/rocketmq/store/AppendMessageCallback.java create mode 100644 rocketmq-store/src/main/java/com/alibaba/rocketmq/store/AppendMessageResult.java create mode 100644 rocketmq-store/src/main/java/com/alibaba/rocketmq/store/AppendMessageStatus.java create mode 100644 rocketmq-store/src/main/java/com/alibaba/rocketmq/store/CommitLog.java create mode 100644 rocketmq-store/src/main/java/com/alibaba/rocketmq/store/ConsumeQueue.java create mode 100644 rocketmq-store/src/main/java/com/alibaba/rocketmq/store/DefaultMessageFilter.java create mode 100644 rocketmq-store/src/main/java/com/alibaba/rocketmq/store/DefaultMessageStore.java create mode 100644 rocketmq-store/src/main/java/com/alibaba/rocketmq/store/DispatchRequest.java create mode 100644 rocketmq-store/src/main/java/com/alibaba/rocketmq/store/GetMessageResult.java create mode 100644 rocketmq-store/src/main/java/com/alibaba/rocketmq/store/GetMessageStatus.java create mode 100644 rocketmq-store/src/main/java/com/alibaba/rocketmq/store/MapedFile.java create mode 100644 rocketmq-store/src/main/java/com/alibaba/rocketmq/store/MapedFileQueue.java create mode 100644 rocketmq-store/src/main/java/com/alibaba/rocketmq/store/MessageExtBrokerInner.java create mode 100644 rocketmq-store/src/main/java/com/alibaba/rocketmq/store/MessageFilter.java create mode 100644 rocketmq-store/src/main/java/com/alibaba/rocketmq/store/MessageStore.java create mode 100644 rocketmq-store/src/main/java/com/alibaba/rocketmq/store/PutMessageResult.java create mode 100644 rocketmq-store/src/main/java/com/alibaba/rocketmq/store/PutMessageStatus.java create mode 100644 rocketmq-store/src/main/java/com/alibaba/rocketmq/store/QueryMessageResult.java create mode 100644 rocketmq-store/src/main/java/com/alibaba/rocketmq/store/ReferenceResource.java create mode 100644 rocketmq-store/src/main/java/com/alibaba/rocketmq/store/RunningFlags.java create mode 100644 rocketmq-store/src/main/java/com/alibaba/rocketmq/store/SelectMapedBufferResult.java create mode 100644 rocketmq-store/src/main/java/com/alibaba/rocketmq/store/StoreCheckpoint.java create mode 100644 rocketmq-store/src/main/java/com/alibaba/rocketmq/store/StoreStatsService.java create mode 100644 rocketmq-store/src/main/java/com/alibaba/rocketmq/store/StoreUtil.java create mode 100644 rocketmq-store/src/main/java/com/alibaba/rocketmq/store/config/BrokerRole.java create mode 100644 rocketmq-store/src/main/java/com/alibaba/rocketmq/store/config/FlushDiskType.java create mode 100644 rocketmq-store/src/main/java/com/alibaba/rocketmq/store/config/MessageStoreConfig.java create mode 100644 rocketmq-store/src/main/java/com/alibaba/rocketmq/store/config/StorePathConfigHelper.java create mode 100644 rocketmq-store/src/main/java/com/alibaba/rocketmq/store/ha/HAConnection.java create mode 100644 rocketmq-store/src/main/java/com/alibaba/rocketmq/store/ha/HAService.java create mode 100644 rocketmq-store/src/main/java/com/alibaba/rocketmq/store/ha/WaitNotifyObject.java create mode 100644 rocketmq-store/src/main/java/com/alibaba/rocketmq/store/index/IndexFile.java create mode 100644 rocketmq-store/src/main/java/com/alibaba/rocketmq/store/index/IndexHeader.java create mode 100644 rocketmq-store/src/main/java/com/alibaba/rocketmq/store/index/IndexService.java create mode 100644 rocketmq-store/src/main/java/com/alibaba/rocketmq/store/index/QueryOffsetResult.java create mode 100644 rocketmq-store/src/main/java/com/alibaba/rocketmq/store/schedule/DelayOffsetSerializeWrapper.java create mode 100644 rocketmq-store/src/main/java/com/alibaba/rocketmq/store/schedule/ScheduleMessageService.java create mode 100644 rocketmq-store/src/main/java/com/alibaba/rocketmq/store/stats/BrokerStats.java create mode 100644 rocketmq-store/src/main/java/com/alibaba/rocketmq/store/stats/BrokerStatsManager.java create mode 100644 rocketmq-store/src/test/java/com/alibaba/rocketmq/store/DefaultMessageStoreTest.java create mode 100644 rocketmq-store/src/test/java/com/alibaba/rocketmq/store/MapedFileQueueTest.java create mode 100644 rocketmq-store/src/test/java/com/alibaba/rocketmq/store/MapedFileTest.java create mode 100644 rocketmq-store/src/test/java/com/alibaba/rocketmq/store/RecoverTest.java create mode 100644 rocketmq-store/src/test/java/com/alibaba/rocketmq/store/StoreCheckpointTest.java create mode 100644 rocketmq-store/src/test/java/com/alibaba/rocketmq/store/index/IndexFileTest.java create mode 100644 rocketmq-store/src/test/java/com/alibaba/rocketmq/store/schedule/ScheduleMessageTest.java create mode 100644 rocketmq-tools/pom.xml create mode 100644 rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/admin/DefaultMQAdminExt.java create mode 100644 rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/admin/DefaultMQAdminExtImpl.java create mode 100644 rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/admin/MQAdminExt.java create mode 100644 rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/admin/api/MessageTrack.java create mode 100644 rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/admin/api/TrackType.java create mode 100644 rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/CommandUtil.java create mode 100644 rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/MQAdminStartup.java create mode 100644 rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/SubCommand.java create mode 100644 rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/broker/BrokerStatusSubCommand.java create mode 100644 rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/broker/CleanExpiredCQSubCommand.java create mode 100644 rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/broker/UpdateBrokerConfigSubCommand.java create mode 100644 rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/cluster/ClusterListSubCommand.java create mode 100644 rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/connection/ConsumerConnectionSubCommand.java create mode 100644 rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/connection/ProducerConnectionSubCommand.java create mode 100644 rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/consumer/ConsumerProgressSubCommand.java create mode 100644 rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/consumer/ConsumerStatusSubCommand.java create mode 100644 rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/consumer/DeleteSubscriptionGroupCommand.java create mode 100644 rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/consumer/StartMonitoringSubCommand.java create mode 100644 rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/consumer/UpdateSubGroupSubCommand.java create mode 100644 rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/message/CheckMsgSubCommand.java create mode 100644 rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/message/PrintMessageSubCommand.java create mode 100644 rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/message/QueryMsgByIdSubCommand.java create mode 100644 rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/message/QueryMsgByKeySubCommand.java create mode 100644 rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/message/QueryMsgByOffsetSubCommand.java create mode 100644 rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/message/Store.java create mode 100644 rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/namesrv/DeleteKvConfigCommand.java create mode 100644 rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/namesrv/DeleteProjectGroupCommand.java create mode 100644 rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/namesrv/GetProjectGroupCommand.java create mode 100644 rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/namesrv/UpdateKvConfigCommand.java create mode 100644 rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/namesrv/UpdateProjectGroupCommand.java create mode 100644 rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/namesrv/WipeWritePermSubCommand.java create mode 100644 rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/offset/CloneGroupOffsetCommand.java create mode 100644 rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/offset/GetConsumerStatusCommand.java create mode 100644 rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/offset/ResetOffsetByTimeCommand.java create mode 100644 rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/offset/ResetOffsetByTimeOldCommand.java create mode 100644 rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/topic/DeleteTopicSubCommand.java create mode 100644 rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/topic/TopicListSubCommand.java create mode 100644 rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/topic/TopicRouteSubCommand.java create mode 100644 rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/topic/TopicStatusSubCommand.java create mode 100644 rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/topic/UpdateOrderConfCommand.java create mode 100644 rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/topic/UpdateTopicSubCommand.java create mode 100644 rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/monitor/DefaultMonitorListener.java create mode 100644 rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/monitor/DeleteMsgsEvent.java create mode 100644 rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/monitor/FailedMsgs.java create mode 100644 rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/monitor/MonitorConfig.java create mode 100644 rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/monitor/MonitorListener.java create mode 100644 rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/monitor/MonitorService.java create mode 100644 rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/monitor/UndoneMsgs.java create mode 100644 test/consumer.sh create mode 100644 test/producer.sh create mode 100644 test/runclass.sh diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..41fd6fb1c --- /dev/null +++ b/.gitignore @@ -0,0 +1,49 @@ +rocketmq-broker/.classpath +rocketmq-broker/.project +rocketmq-broker/.settings/org.eclipse.jdt.core.prefs +rocketmq-broker/target/classes/.gitignore +rocketmq-broker/target/test-classes/.gitignore +rocketmq-client/.classpath +rocketmq-client/.project +rocketmq-client/.settings/org.eclipse.jdt.core.prefs +rocketmq-client/target/classes/.gitignore +rocketmq-client/target/test-classes/.gitignore +rocketmq-common/.classpath +rocketmq-common/.project +rocketmq-common/.settings/org.eclipse.jdt.core.prefs +rocketmq-common/target/classes/.gitignore +rocketmq-common/target/test-classes/.gitignore +rocketmq-example/.classpath +rocketmq-example/.project +rocketmq-example/.settings/org.eclipse.jdt.core.prefs +rocketmq-example/target/classes/.gitignore +rocketmq-filtersrv/.classpath +rocketmq-filtersrv/.project +rocketmq-filtersrv/.settings/org.eclipse.jdt.core.prefs +rocketmq-filtersrv/target/.gitignore +rocketmq-namesrv/.classpath +rocketmq-namesrv/.project +rocketmq-namesrv/.settings/org.eclipse.jdt.core.prefs +rocketmq-namesrv/target/.gitignore +rocketmq-remoting/.classpath +rocketmq-remoting/.project +rocketmq-remoting/.settings/org.eclipse.jdt.core.prefs +rocketmq-remoting/target/classes/.gitignore +rocketmq-remoting/target/test-classes/.gitignore +rocketmq-srvutil/.classpath +rocketmq-srvutil/.project +rocketmq-srvutil/.settings/org.eclipse.jdt.core.prefs +rocketmq-srvutil/target/.gitignore +rocketmq-srvutil/target/test-classes/.gitignore +rocketmq-store/.classpath +rocketmq-store/.project +rocketmq-store/.settings/org.eclipse.jdt.core.prefs +rocketmq-store/target/.gitignore +rocketmq-store/target/test-classes/.gitignore +rocketmq-tools/.classpath +rocketmq-tools/.project +rocketmq-tools/.settings/org.eclipse.jdt.core.prefs +rocketmq-tools/target/.gitignore +target/mvn-eclipse-cache.properties +*.class +target/ diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 000000000..dff5f3a5d --- /dev/null +++ b/.travis.yml @@ -0,0 +1 @@ +language: java diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 000000000..d9a95f5c2 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,191 @@ +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, and +distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the copyright +owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other entities +that control, are controlled by, or are under common control with that entity. +For the purposes of this definition, "control" means (i) the power, direct or +indirect, to cause the direction or management of such entity, whether by +contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the +outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising +permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, including +but not limited to software source code, documentation source, and configuration +files. + +"Object" form shall mean any form resulting from mechanical transformation or +translation of a Source form, including but not limited to compiled object code, +generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, made +available under the License, as indicated by a copyright notice that is included +in or attached to the work (an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, that +is based on (or derived from) the Work and for which the editorial revisions, +annotations, elaborations, or other modifications represent, as a whole, an +original work of authorship. For the purposes of this License, Derivative Works +shall not include works that remain separable from, or merely link (or bind by +name) to the interfaces of, the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including the original version +of the Work and any modifications or additions to that Work or Derivative Works +thereof, that is intentionally submitted to Licensor for inclusion in the Work +by the copyright owner or by an individual or Legal Entity authorized to submit +on behalf of the copyright owner. For the purposes of this definition, +"submitted" means any form of electronic, verbal, or written communication sent +to the Licensor or its representatives, including but not limited to +communication on electronic mailing lists, source code control systems, and +issue tracking systems that are managed by, or on behalf of, the Licensor for +the purpose of discussing and improving the Work, but excluding communication +that is conspicuously marked or otherwise designated in writing by the copyright +owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf +of whom a Contribution has been received by Licensor and subsequently +incorporated within the Work. + +2. Grant of Copyright License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable copyright license to reproduce, prepare Derivative Works of, +publicly display, publicly perform, sublicense, and distribute the Work and such +Derivative Works in Source or Object form. + +3. Grant of Patent License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable (except as stated in this section) patent license to make, have +made, use, offer to sell, sell, import, and otherwise transfer the Work, where +such license applies only to those patent claims licensable by such Contributor +that are necessarily infringed by their Contribution(s) alone or by combination +of their Contribution(s) with the Work to which such Contribution(s) was +submitted. If You institute patent litigation against any entity (including a +cross-claim or counterclaim in a lawsuit) alleging that the Work or a +Contribution incorporated within the Work constitutes direct or contributory +patent infringement, then any patent licenses granted to You under this License +for that Work shall terminate as of the date such litigation is filed. + +4. Redistribution. + +You may reproduce and distribute copies of the Work or Derivative Works thereof +in any medium, with or without modifications, and in Source or Object form, +provided that You meet the following conditions: + +You must give any other recipients of the Work or Derivative Works a copy of +this License; and +You must cause any modified files to carry prominent notices stating that You +changed the files; and +You must retain, in the Source form of any Derivative Works that You distribute, +all copyright, patent, trademark, and attribution notices from the Source form +of the Work, excluding those notices that do not pertain to any part of the +Derivative Works; and +If the Work includes a "NOTICE" text file as part of its distribution, then any +Derivative Works that You distribute must include a readable copy of the +attribution notices contained within such NOTICE file, excluding those notices +that do not pertain to any part of the Derivative Works, in at least one of the +following places: within a NOTICE text file distributed as part of the +Derivative Works; within the Source form or documentation, if provided along +with the Derivative Works; or, within a display generated by the Derivative +Works, if and wherever such third-party notices normally appear. The contents of +the NOTICE file are for informational purposes only and do not modify the +License. You may add Your own attribution notices within Derivative Works that +You distribute, alongside or as an addendum to the NOTICE text from the Work, +provided that such additional attribution notices cannot be construed as +modifying the License. +You may add Your own copyright statement to Your modifications and may provide +additional or different license terms and conditions for use, reproduction, or +distribution of Your modifications, or for any such Derivative Works as a whole, +provided Your use, reproduction, and distribution of the Work otherwise complies +with the conditions stated in this License. + +5. Submission of Contributions. + +Unless You explicitly state otherwise, any Contribution intentionally submitted +for inclusion in the Work by You to the Licensor shall be under the terms and +conditions of this License, without any additional terms or conditions. +Notwithstanding the above, nothing herein shall supersede or modify the terms of +any separate license agreement you may have executed with Licensor regarding +such Contributions. + +6. Trademarks. + +This License does not grant permission to use the trade names, trademarks, +service marks, or product names of the Licensor, except as required for +reasonable and customary use in describing the origin of the Work and +reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. + +Unless required by applicable law or agreed to in writing, Licensor provides the +Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, +including, without limitation, any warranties or conditions of TITLE, +NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are +solely responsible for determining the appropriateness of using or +redistributing the Work and assume any risks associated with Your exercise of +permissions under this License. + +8. Limitation of Liability. + +In no event and under no legal theory, whether in tort (including negligence), +contract, or otherwise, unless required by applicable law (such as deliberate +and grossly negligent acts) or agreed to in writing, shall any Contributor be +liable to You for damages, including any direct, indirect, special, incidental, +or consequential damages of any character arising as a result of this License or +out of the use or inability to use the Work (including but not limited to +damages for loss of goodwill, work stoppage, computer failure or malfunction, or +any and all other commercial damages or losses), even if such Contributor has +been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. + +While redistributing the Work or Derivative Works thereof, You may choose to +offer, and charge a fee for, acceptance of support, warranty, indemnity, or +other liability obligations and/or rights consistent with this License. However, +in accepting such obligations, You may act only on Your own behalf and on Your +sole responsibility, not on behalf of any other Contributor, and only if You +agree to indemnify, defend, and hold each Contributor harmless for any liability +incurred by, or claims asserted against, such Contributor by reason of your +accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work + +To apply the Apache License to your work, attach the following boilerplate +notice, with the fields enclosed by brackets "[]" replaced with your own +identifying information. (Don't include the brackets!) The text should be +enclosed in the appropriate comment syntax for the file format. We also +recommend that a file or class name and description of purpose be included on +the same "printed page" as the copyright notice for easier identification within +third-party archives. + + Copyright 2013 Alibaba Group Holding Limited + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md index a65d1df5f..5a9b39305 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,41 @@ -RocketMQ -======== +### RocketMQ是什么？[![Build Status](https://travis-ci.org/alibaba/RocketMQ.svg?branch=develop)](https://travis-ci.org/alibaba/RocketMQ) +RocketMQ是一款分布式、队列模型的消息中间件，具有以下特点： -Alibaba's MQ, also aliyun ONS. +* 支持严格的消息顺序 +* 支持Topic与Queue两种模式 +* 亿级消息堆积能力 +* 比较友好的分布式特性 +* 同时支持Push与Pull方式消费消息 +* 历经多次天猫双十一海量消息考验 +---------- -### 源码即将开放。 +### 如何开始？`必读` +* [下载最新版安装包](https://github.com/alibaba/RocketMQ/releases) +* [向开发者索要最新文档](https://github.com/alibaba/RocketMQ/issues/1) +* [`在阿里云上使用RocketMQ`](http://onsall.oss-cn-hangzhou.aliyuncs.com/ALIYUN_ONS_USER_GUIDE.pdf)，可以免去繁琐的部署运维、成本。 + +---------- + +### 开源协议 +[Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0.html) Copyright (C) 2010-2013 Alibaba Group Holding Limited + +---------- + +### 开发规范 +* 代码使用Eclipse代码样式格式化，提交代码前须格式化[rocketmq.java.code.style.xml](https://github.com/alibaba/RocketMQ/blob/master/docs/rocketmq.java.code.style.xml) +* Java源文件使用Unix换行、UTF-8文件编码 +* 请在git clone命令之前执行`git config --global core.autocrlf false`，确保本地代码使用Unix换行格式 +* 请在develop分支上开发 +* 每次提交须有Issue关联，可在git提交注释中增加`#issue号码`进行关联 + +---------- + +### 联系我们 + +* [向我们提交建议、BUG、寻求技术帮助](https://github.com/alibaba/RocketMQ/issues/new) +* 欢迎参与RocketMQ项目，只需在Github上fork、pull request即可。 +* [到新浪微博交流RocketMQ](http://q.weibo.com/1628465) +* 加入QQ群交流，[5776652](http://url.cn/Knxm0o) + +---------- diff --git a/benchmark/consumer.sh b/benchmark/consumer.sh new file mode 100644 index 000000000..a94a617d2 --- /dev/null +++ b/benchmark/consumer.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +# +# $Id: consumer.sh 1831 2013-05-16 01:39:51Z shijia.wxr $ +# +sh ./runclass.sh com.alibaba.rocketmq.example.benchmark.Consumer $@ diff --git a/benchmark/producer.sh b/benchmark/producer.sh new file mode 100644 index 000000000..27fab9c22 --- /dev/null +++ b/benchmark/producer.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +# +# $Id: producer.sh 1831 2013-05-16 01:39:51Z shijia.wxr $ +# +sh ./runclass.sh -Dcom.alibaba.rocketmq.client.sendSmartMsg=true com.alibaba.rocketmq.example.benchmark.Producer $@ diff --git a/benchmark/runclass.sh b/benchmark/runclass.sh new file mode 100644 index 000000000..e87c919c8 --- /dev/null +++ b/benchmark/runclass.sh @@ -0,0 +1,30 @@ +#!/bin/sh + +# +# $Id: runserver.sh 1831 2013-05-16 01:39:51Z shijia.wxr $ +# + +if [ $# -lt 1 ]; +then + echo "USAGE: $0 classname opts" + exit 1 +fi + +BASE_DIR=$(dirname $0)/.. +CLASSPATH=.:${BASE_DIR}/conf:${CLASSPATH} + +JAVA_OPT="${JAVA_OPT} -server -Xms1g -Xmx1g -Xmn256m -XX:PermSize=128m -XX:MaxPermSize=320m" +JAVA_OPT="${JAVA_OPT} -XX:+UseConcMarkSweepGC -XX:+UseCMSCompactAtFullCollection -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled -XX:SoftRefLRUPolicyMSPerMB=0 -XX:+CMSClassUnloadingEnabled -XX:SurvivorRatio=8 -XX:+DisableExplicitGC" +JAVA_OPT="${JAVA_OPT} -verbose:gc -Xloggc:${HOME}/rmq_srv_gc.log -XX:+PrintGCDetails" +JAVA_OPT="${JAVA_OPT} -XX:-OmitStackTraceInFastThrow" +JAVA_OPT="${JAVA_OPT} -Djava.ext.dirs=${BASE_DIR}/lib" +#JAVA_OPT="${JAVA_OPT} -Xdebug -Xrunjdwp:transport=dt_socket,address=9555,server=y,suspend=n" +JAVA_OPT="${JAVA_OPT} -cp ${CLASSPATH}" + +if [ -z "$JAVA_HOME" ]; then + JAVA_HOME=/opt/taobao/java +fi + +JAVA="$JAVA_HOME/bin/java" + +$JAVA ${JAVA_OPT} $@ diff --git a/benchmark/tproducer.sh b/benchmark/tproducer.sh new file mode 100644 index 000000000..72043854e --- /dev/null +++ b/benchmark/tproducer.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +# +# $Id: producer.sh 1831 2013-05-16 01:39:51Z shijia.wxr $ +# +sh ./runclass.sh com.alibaba.rocketmq.example.benchmark.TransactionProducer $@ diff --git a/bin/README.md b/bin/README.md new file mode 100644 index 000000000..58903a118 --- /dev/null +++ b/bin/README.md @@ -0,0 +1,34 @@ +### 操作系统调优 +在生产环境部署Broker前，必须要执行os.sh，对操作系统进行调优 + +**P.S: os.sh只能执行一次，需要sudo root权限** + +### 启动broker +* Unix平台 + + `nohup sh mqbroker &` + +* Windows平台（仅支持64位） + + `mqbroker.exe` + +### 关闭broker + sh mqshutdown broker + +### 启动Name Server +* Unix平台 + + `nohup sh mqnamesrv &` + +* Windows平台（仅支持64位） + + `mqnamesrv.exe` + +### 关闭Name Server + sh mqshutdown namesrv + +### 更新或创建Topic + sh mqadmin updateTopic -b 127.0.0.1:10911 -t TopicA + +### 更新或创建订阅组 + sh mqadmin updateSubGroup -b 127.0.0.1:10911 -g SubGroupA \ No newline at end of file diff --git a/bin/mqadmin b/bin/mqadmin new file mode 100644 index 000000000..67a94c7bb --- /dev/null +++ b/bin/mqadmin @@ -0,0 +1,34 @@ +#!/bin/sh + +# +# $Id: mqbroker 1840 2013-05-16 02:13:59Z shijia.wxr $ +# + +if [ -z "$ROCKETMQ_HOME" ] ; then + ## resolve links - $0 may be a link to maven's home + PRG="$0" + + # need this for relative symlinks + while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG="`dirname "$PRG"`/$link" + fi + done + + saveddir=`pwd` + + ROCKETMQ_HOME=`dirname "$PRG"`/.. + + # make it fully qualified + ROCKETMQ_HOME=`cd "$ROCKETMQ_HOME" && pwd` + + cd "$saveddir" +fi + +export ROCKETMQ_HOME + +sh ${ROCKETMQ_HOME}/bin/tools.sh com.alibaba.rocketmq.tools.command.MQAdminStartup $@ diff --git a/bin/mqadmin.xml b/bin/mqadmin.xml new file mode 100644 index 000000000..560a8e5ae --- /dev/null +++ b/bin/mqadmin.xml @@ -0,0 +1,26 @@ + + false + + ${JAVA_HOME} + + server + + com.alibaba.rocketmq.tools.command.MQAdminStartup + + + ${cpd}/../lib + ${cpd}/.. + + + + + + + <-Xms512m> + <-Xmx1g> + <-XX:NewSize>256M + <-XX:MaxNewSize>512M + <-XX:PermSize>128M + <-XX:MaxPermSize>128M + + diff --git a/bin/mqbroker b/bin/mqbroker new file mode 100644 index 000000000..32b88ba6d --- /dev/null +++ b/bin/mqbroker @@ -0,0 +1,34 @@ +#!/bin/sh + +# +# $Id: mqbroker 1840 2013-05-16 02:13:59Z shijia.wxr $ +# + +if [ -z "$ROCKETMQ_HOME" ] ; then + ## resolve links - $0 may be a link to maven's home + PRG="$0" + + # need this for relative symlinks + while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG="`dirname "$PRG"`/$link" + fi + done + + saveddir=`pwd` + + ROCKETMQ_HOME=`dirname "$PRG"`/.. + + # make it fully qualified + ROCKETMQ_HOME=`cd "$ROCKETMQ_HOME" && pwd` + + cd "$saveddir" +fi + +export ROCKETMQ_HOME + +sh ${ROCKETMQ_HOME}/bin/runbroker.sh com.alibaba.rocketmq.broker.BrokerStartup $@ diff --git a/bin/mqbroker.xml b/bin/mqbroker.xml new file mode 100644 index 000000000..0cd31d6b4 --- /dev/null +++ b/bin/mqbroker.xml @@ -0,0 +1,26 @@ + + false + + ${JAVA_HOME} + + server + + com.alibaba.rocketmq.broker.BrokerStartup + + + ${cpd}/../lib + ${cpd}/.. + + + + + + + <-Xms512m> + <-Xmx1g> + <-XX:NewSize>256M + <-XX:MaxNewSize>512M + <-XX:PermSize>128M + <-XX:MaxPermSize>128M + + diff --git a/bin/mqfiltersrv b/bin/mqfiltersrv new file mode 100644 index 000000000..669a1fd90 --- /dev/null +++ b/bin/mqfiltersrv @@ -0,0 +1,34 @@ +#!/bin/sh + +# +# $Id: mqbroker 587 2012-11-20 03:26:56Z shijia.wxr $ +# + +if [ -z "$ROCKETMQ_HOME" ] ; then + ## resolve links - $0 may be a link to maven's home + PRG="$0" + + # need this for relative symlinks + while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG="`dirname "$PRG"`/$link" + fi + done + + saveddir=`pwd` + + ROCKETMQ_HOME=`dirname "$PRG"`/.. + + # make it fully qualified + ROCKETMQ_HOME=`cd "$ROCKETMQ_HOME" && pwd` + + cd "$saveddir" +fi + +export ROCKETMQ_HOME + +sh ${ROCKETMQ_HOME}/bin/runserver.sh com.alibaba.rocketmq.filtersrv.FiltersrvStartup $@ diff --git a/bin/mqfiltersrv.xml b/bin/mqfiltersrv.xml new file mode 100644 index 000000000..ea0920447 --- /dev/null +++ b/bin/mqfiltersrv.xml @@ -0,0 +1,26 @@ + + false + + ${JAVA_HOME} + + server + + com.alibaba.rocketmq.filtersrv.FiltersrvStartup + + + ${cpd}/../lib + ${cpd}/.. + + + + + + + <-Xms512m> + <-Xmx1g> + <-XX:NewSize>256M + <-XX:MaxNewSize>512M + <-XX:PermSize>128M + <-XX:MaxPermSize>128M + + diff --git a/bin/mqnamesrv b/bin/mqnamesrv new file mode 100644 index 000000000..e082e889f --- /dev/null +++ b/bin/mqnamesrv @@ -0,0 +1,34 @@ +#!/bin/sh + +# +# $Id: mqbroker 587 2012-11-20 03:26:56Z shijia.wxr $ +# + +if [ -z "$ROCKETMQ_HOME" ] ; then + ## resolve links - $0 may be a link to maven's home + PRG="$0" + + # need this for relative symlinks + while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG="`dirname "$PRG"`/$link" + fi + done + + saveddir=`pwd` + + ROCKETMQ_HOME=`dirname "$PRG"`/.. + + # make it fully qualified + ROCKETMQ_HOME=`cd "$ROCKETMQ_HOME" && pwd` + + cd "$saveddir" +fi + +export ROCKETMQ_HOME + +sh ${ROCKETMQ_HOME}/bin/runserver.sh com.alibaba.rocketmq.namesrv.NamesrvStartup $@ diff --git a/bin/mqnamesrv.xml b/bin/mqnamesrv.xml new file mode 100644 index 000000000..c9701ccf7 --- /dev/null +++ b/bin/mqnamesrv.xml @@ -0,0 +1,26 @@ + + false + + ${JAVA_HOME} + + server + + com.alibaba.rocketmq.namesrv.NamesrvStartup + + + ${cpd}/../lib + ${cpd}/.. + + + + + + + <-Xms512m> + <-Xmx1g> + <-XX:NewSize>256M + <-XX:MaxNewSize>512M + <-XX:PermSize>128M + <-XX:MaxPermSize>128M + + diff --git a/bin/mqshutdown b/bin/mqshutdown new file mode 100644 index 000000000..6a3b6ff73 --- /dev/null +++ b/bin/mqshutdown @@ -0,0 +1,34 @@ +#!/bin/sh + +case $1 in + broker) + + pid=`ps ax | grep -i 'com.alibaba.rocketmq.broker.BrokerStartup' |grep java | grep -v grep | awk '{print $1}'` + if [ -z "$pid" ] ; then + echo "No mqbroker running." + exit -1; + fi + + echo "The mqbroker(${pid}) is running..." + + kill ${pid} + + echo "Send shutdown request to mqbroker(${pid}) OK" + ;; + namesrv) + + pid=`ps ax | grep -i 'com.alibaba.rocketmq.namesrv.NamesrvStartup' |grep java | grep -v grep | awk '{print $1}'` + if [ -z "$pid" ] ; then + echo "No mqnamesrv running." + exit -1; + fi + + echo "The mqnamesrv(${pid}) is running..." + + kill ${pid} + + echo "Send shutdown request to mqnamesrv(${pid}) OK" + ;; + *) + echo "Useage: mqshutdown broker | namesrv" +esac diff --git a/bin/os.sh b/bin/os.sh new file mode 100644 index 000000000..372a0bb2c --- /dev/null +++ b/bin/os.sh @@ -0,0 +1,40 @@ +#!/bin/sh + +# +# Execute Only Once +# + +echo 'vm.overcommit_memory=1' >> /etc/sysctl.conf +echo 'vm.min_free_kbytes=5000000' >> /etc/sysctl.conf +echo 'vm.drop_caches=1' >> /etc/sysctl.conf +echo 'vm.zone_reclaim_mode=0' >> /etc/sysctl.conf +echo 'vm.max_map_count=655360' >> /etc/sysctl.conf +echo 'vm.dirty_background_ratio=50' >> /etc/sysctl.conf +echo 'vm.dirty_ratio=50' >> /etc/sysctl.conf +echo 'vm.page-cluster=3' >> /etc/sysctl.conf +echo 'vm.dirty_writeback_centisecs=360000' >> /etc/sysctl.conf +echo 'vm.swappiness=10' >> /etc/sysctl.conf +sysctl -p + +echo 'ulimit -n 655350' >> /etc/profile +echo 'admin hard nofile 655350' >> /etc/security/limits.conf + +DISK=`df -k | sort -n -r -k 2 | awk -F/ 'NR==1 {gsub(/[0-9].*/,"",$3); print $3}'` +[ "$DISK" = 'cciss' ] && DISK='cciss!c0d0' +echo 'deadline' > /sys/block/$DISK/queue/scheduler + + +echo "---------------------------------------------------------------" +sysctl vm.overcommit_memory +sysctl vm.min_free_kbytes +sysctl vm.drop_caches +sysctl vm.zone_reclaim_mode +sysctl vm.max_map_count +sysctl vm.dirty_background_ratio +sysctl vm.dirty_ratio +sysctl vm.page-cluster +sysctl vm.dirty_writeback_centisecs +sysctl vm.swappiness + +su - admin -c 'ulimit -n' +cat /sys/block/$DISK/queue/scheduler diff --git a/bin/play.sh b/bin/play.sh new file mode 100644 index 000000000..4b150c24e --- /dev/null +++ b/bin/play.sh @@ -0,0 +1,18 @@ +#!/bin/sh + +# +# Name Server +# +nohup sh mqnamesrv > ns.log 2>&1 & + +# +# Service Addr +# +ADDR=`hostname -i`:9876 + +# +# Broker +# +nohup sh mqbroker -n ${ADDR} > bk.log 2>&1 & + +echo "Start Name Server and Broker Successfully, ${ADDR}" diff --git a/bin/runbroker.sh b/bin/runbroker.sh new file mode 100644 index 000000000..bf6de7e05 --- /dev/null +++ b/bin/runbroker.sh @@ -0,0 +1,38 @@ +#!/bin/sh + +#=========================================================================================== +# Java 环境设置 +#=========================================================================================== +error_exit () +{ + echo "ERROR: $1 !!" + exit 1 +} + +[ ! -e "$JAVA_HOME/bin/java" ] && JAVA_HOME=$HOME/jdk/java +[ ! -e "$JAVA_HOME/bin/java" ] && JAVA_HOME=/opt/taobao/java +[ ! -e "$JAVA_HOME/bin/java" ] && error_exit "Please set the JAVA_HOME variable in your environment, We need java(x64)!" + +export JAVA_HOME +export JAVA="$JAVA_HOME/bin/java" +export BASE_DIR=$(dirname $0)/.. +export CLASSPATH=.:${BASE_DIR}/conf:${CLASSPATH} + +#=========================================================================================== +# JVM 参数配置 +#=========================================================================================== +JAVA_OPT="${JAVA_OPT} -server -Xms4g -Xmx4g -Xmn2g -XX:PermSize=128m -XX:MaxPermSize=320m" +JAVA_OPT="${JAVA_OPT} -XX:+UseConcMarkSweepGC -XX:+UseCMSCompactAtFullCollection -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled -XX:SoftRefLRUPolicyMSPerMB=0 -XX:+CMSClassUnloadingEnabled -XX:SurvivorRatio=8 -XX:+DisableExplicitGC" +JAVA_OPT="${JAVA_OPT} -verbose:gc -Xloggc:${HOME}/rmq_bk_gc.log -XX:+PrintGCDetails -XX:+PrintGCDateStamps" +JAVA_OPT="${JAVA_OPT} -XX:-OmitStackTraceInFastThrow" +JAVA_OPT="${JAVA_OPT} -Djava.ext.dirs=${BASE_DIR}/lib" +#JAVA_OPT="${JAVA_OPT} -Xdebug -Xrunjdwp:transport=dt_socket,address=9555,server=y,suspend=n" +JAVA_OPT="${JAVA_OPT} -cp ${CLASSPATH}" + +numactl --interleave=all pwd > /dev/null 2>&1 +if [ $? -eq 0 ] +then + numactl --interleave=all $JAVA ${JAVA_OPT} $@ +else + $JAVA ${JAVA_OPT} $@ +fi diff --git a/bin/runserver.sh b/bin/runserver.sh new file mode 100644 index 000000000..4200f938a --- /dev/null +++ b/bin/runserver.sh @@ -0,0 +1,32 @@ +#!/bin/sh + +#=========================================================================================== +# Java 环境设置 +#=========================================================================================== +error_exit () +{ + echo "ERROR: $1 !!" + exit 1 +} + +[ ! -e "$JAVA_HOME/bin/java" ] && JAVA_HOME=$HOME/jdk/java +[ ! -e "$JAVA_HOME/bin/java" ] && JAVA_HOME=/opt/taobao/java +[ ! -e "$JAVA_HOME/bin/java" ] && error_exit "Please set the JAVA_HOME variable in your environment, We need java(x64)!" + +export JAVA_HOME +export JAVA="$JAVA_HOME/bin/java" +export BASE_DIR=$(dirname $0)/.. +export CLASSPATH=.:${BASE_DIR}/conf:${CLASSPATH} + +#=========================================================================================== +# JVM 参数配置 +#=========================================================================================== +JAVA_OPT="${JAVA_OPT} -server -Xms4g -Xmx4g -Xmn2g -XX:PermSize=128m -XX:MaxPermSize=320m" +JAVA_OPT="${JAVA_OPT} -XX:+UseConcMarkSweepGC -XX:+UseCMSCompactAtFullCollection -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled -XX:SoftRefLRUPolicyMSPerMB=0 -XX:+CMSClassUnloadingEnabled -XX:SurvivorRatio=8 -XX:+DisableExplicitGC" +JAVA_OPT="${JAVA_OPT} -verbose:gc -Xloggc:${HOME}/rmq_srv_gc.log -XX:+PrintGCDetails" +JAVA_OPT="${JAVA_OPT} -XX:-OmitStackTraceInFastThrow" +JAVA_OPT="${JAVA_OPT} -Djava.ext.dirs=${BASE_DIR}/lib" +#JAVA_OPT="${JAVA_OPT} -Xdebug -Xrunjdwp:transport=dt_socket,address=9555,server=y,suspend=n" +JAVA_OPT="${JAVA_OPT} -cp ${CLASSPATH}" + +$JAVA ${JAVA_OPT} $@ diff --git a/bin/startfsrv.sh b/bin/startfsrv.sh new file mode 100644 index 000000000..5481382cc --- /dev/null +++ b/bin/startfsrv.sh @@ -0,0 +1,34 @@ +#!/bin/sh + +# +# $Id: mqbroker 587 2012-11-20 03:26:56Z shijia.wxr $ +# + +if [ -z "$ROCKETMQ_HOME" ] ; then + ## resolve links - $0 may be a link to maven's home + PRG="$0" + + # need this for relative symlinks + while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG="`dirname "$PRG"`/$link" + fi + done + + saveddir=`pwd` + + ROCKETMQ_HOME=`dirname "$PRG"`/.. + + # make it fully qualified + ROCKETMQ_HOME=`cd "$ROCKETMQ_HOME" && pwd` + + cd "$saveddir" +fi + +export ROCKETMQ_HOME + +nohup sh ${ROCKETMQ_HOME}/bin/runserver.sh com.alibaba.rocketmq.filtersrv.FiltersrvStartup $@ & diff --git a/bin/tools.sh b/bin/tools.sh new file mode 100644 index 000000000..aa2696a67 --- /dev/null +++ b/bin/tools.sh @@ -0,0 +1,28 @@ +#!/bin/sh + +#=========================================================================================== +# Java 环境设置 +#=========================================================================================== +error_exit () +{ + echo "ERROR: $1 !!" + exit 1 +} + +[ ! -e "$JAVA_HOME/bin/java" ] && JAVA_HOME=$HOME/jdk/java +[ ! -e "$JAVA_HOME/bin/java" ] && JAVA_HOME=/opt/taobao/java +[ ! -e "$JAVA_HOME/bin/java" ] && error_exit "Please set the JAVA_HOME variable in your environment, We need java(x64)!" + +export JAVA_HOME +export JAVA="$JAVA_HOME/bin/java" +export BASE_DIR=$(dirname $0)/.. +export CLASSPATH=.:${BASE_DIR}/conf:${CLASSPATH} + +#=========================================================================================== +# JVM 参数配置 +#=========================================================================================== +JAVA_OPT="${JAVA_OPT} -server -Xms1g -Xmx1g -Xmn256m -XX:PermSize=128m -XX:MaxPermSize=128m" +JAVA_OPT="${JAVA_OPT} -Djava.ext.dirs=${BASE_DIR}/lib" +JAVA_OPT="${JAVA_OPT} -cp ${CLASSPATH}" + +$JAVA ${JAVA_OPT} $@ diff --git a/conf/2m-2s-async/broker-a-s.properties b/conf/2m-2s-async/broker-a-s.properties new file mode 100644 index 000000000..28d8923c3 --- /dev/null +++ b/conf/2m-2s-async/broker-a-s.properties @@ -0,0 +1,7 @@ +brokerClusterName=DefaultCluster +brokerName=broker-a +brokerId=1 +deleteWhen=04 +fileReservedTime=48 +brokerRole=SLAVE +flushDiskType=ASYNC_FLUSH diff --git a/conf/2m-2s-async/broker-a.properties b/conf/2m-2s-async/broker-a.properties new file mode 100644 index 000000000..a87d0a3c1 --- /dev/null +++ b/conf/2m-2s-async/broker-a.properties @@ -0,0 +1,7 @@ +brokerClusterName=DefaultCluster +brokerName=broker-a +brokerId=0 +deleteWhen=04 +fileReservedTime=48 +brokerRole=ASYNC_MASTER +flushDiskType=ASYNC_FLUSH diff --git a/conf/2m-2s-async/broker-b-s.properties b/conf/2m-2s-async/broker-b-s.properties new file mode 100644 index 000000000..8aca4a301 --- /dev/null +++ b/conf/2m-2s-async/broker-b-s.properties @@ -0,0 +1,7 @@ +brokerClusterName=DefaultCluster +brokerName=broker-b +brokerId=1 +deleteWhen=04 +fileReservedTime=48 +brokerRole=SLAVE +flushDiskType=ASYNC_FLUSH diff --git a/conf/2m-2s-async/broker-b.properties b/conf/2m-2s-async/broker-b.properties new file mode 100644 index 000000000..eaa4d2f47 --- /dev/null +++ b/conf/2m-2s-async/broker-b.properties @@ -0,0 +1,7 @@ +brokerClusterName=DefaultCluster +brokerName=broker-b +brokerId=0 +deleteWhen=04 +fileReservedTime=48 +brokerRole=ASYNC_MASTER +flushDiskType=ASYNC_FLUSH diff --git a/conf/2m-2s-sync/broker-a-s.properties b/conf/2m-2s-sync/broker-a-s.properties new file mode 100644 index 000000000..28d8923c3 --- /dev/null +++ b/conf/2m-2s-sync/broker-a-s.properties @@ -0,0 +1,7 @@ +brokerClusterName=DefaultCluster +brokerName=broker-a +brokerId=1 +deleteWhen=04 +fileReservedTime=48 +brokerRole=SLAVE +flushDiskType=ASYNC_FLUSH diff --git a/conf/2m-2s-sync/broker-a.properties b/conf/2m-2s-sync/broker-a.properties new file mode 100644 index 000000000..41ac2cf18 --- /dev/null +++ b/conf/2m-2s-sync/broker-a.properties @@ -0,0 +1,7 @@ +brokerClusterName=DefaultCluster +brokerName=broker-a +brokerId=0 +deleteWhen=04 +fileReservedTime=48 +brokerRole=SYNC_MASTER +flushDiskType=ASYNC_FLUSH diff --git a/conf/2m-2s-sync/broker-b-s.properties b/conf/2m-2s-sync/broker-b-s.properties new file mode 100644 index 000000000..8aca4a301 --- /dev/null +++ b/conf/2m-2s-sync/broker-b-s.properties @@ -0,0 +1,7 @@ +brokerClusterName=DefaultCluster +brokerName=broker-b +brokerId=1 +deleteWhen=04 +fileReservedTime=48 +brokerRole=SLAVE +flushDiskType=ASYNC_FLUSH diff --git a/conf/2m-2s-sync/broker-b.properties b/conf/2m-2s-sync/broker-b.properties new file mode 100644 index 000000000..4fe6a7f3c --- /dev/null +++ b/conf/2m-2s-sync/broker-b.properties @@ -0,0 +1,7 @@ +brokerClusterName=DefaultCluster +brokerName=broker-b +brokerId=0 +deleteWhen=04 +fileReservedTime=48 +brokerRole=SYNC_MASTER +flushDiskType=ASYNC_FLUSH diff --git a/conf/2m-noslave/broker-a.properties b/conf/2m-noslave/broker-a.properties new file mode 100644 index 000000000..a87d0a3c1 --- /dev/null +++ b/conf/2m-noslave/broker-a.properties @@ -0,0 +1,7 @@ +brokerClusterName=DefaultCluster +brokerName=broker-a +brokerId=0 +deleteWhen=04 +fileReservedTime=48 +brokerRole=ASYNC_MASTER +flushDiskType=ASYNC_FLUSH diff --git a/conf/2m-noslave/broker-b.properties b/conf/2m-noslave/broker-b.properties new file mode 100644 index 000000000..eaa4d2f47 --- /dev/null +++ b/conf/2m-noslave/broker-b.properties @@ -0,0 +1,7 @@ +brokerClusterName=DefaultCluster +brokerName=broker-b +brokerId=0 +deleteWhen=04 +fileReservedTime=48 +brokerRole=ASYNC_MASTER +flushDiskType=ASYNC_FLUSH diff --git a/conf/logback_broker.xml b/conf/logback_broker.xml new file mode 100644 index 000000000..c371c6e1f --- /dev/null +++ b/conf/logback_broker.xml @@ -0,0 +1,217 @@ + + + + ${user.home}/logs/rocketmqlogs/broker_default.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/broker_default.%i.log + + 1 + 5 + + + 100MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n + UTF-8 + + + + + + ${user.home}/logs/rocketmqlogs/broker.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/broker.%i.log + + 1 + 30 + + + 100MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n + UTF-8 + + + + + ${user.home}/logs/rocketmqlogs/store.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/store.%i.log + + 1 + 30 + + + 100MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n + UTF-8 + + + + + ${user.home}/logs/rocketmqlogs/remoting.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/remoting.%i.log + + 1 + 30 + + + 100MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n + UTF-8 + + + + + ${user.home}/logs/rocketmqlogs/storeerror.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/storeerror.%i.log + + 1 + 30 + + + 100MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n + UTF-8 + + + + + + ${user.home}/logs/rocketmqlogs/transaction.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/transaction.%i.log + + 1 + 5 + + + 100MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n + UTF-8 + + + + + + ${user.home}/logs/rocketmqlogs/lock.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/lock.%i.log + + 1 + 5 + + + 100MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n + UTF-8 + + + + + ${user.home}/logs/rocketmqlogs/stats.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/stats-%d{yyyy-MM-dd}.%i.log + + + 500MB + + 10 + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} %p - %m%n + UTF-8 + + + + + true + + %d{yyy-MM-dd HH\:mm\:ss,GMT+8} %p %t - %m%n + UTF-8 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/conf/logback_filtersrv.xml b/conf/logback_filtersrv.xml new file mode 100644 index 000000000..1b4981c83 --- /dev/null +++ b/conf/logback_filtersrv.xml @@ -0,0 +1,70 @@ + + + + ${user.home}/logs/rocketmqlogs/filtersrv_default.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/filtersrv_default.%i.log + + 1 + 5 + + + 100MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n + UTF-8 + + + + + ${user.home}/logs/rocketmqlogs/filtersrv.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/filtersrv.%i.log + + 1 + 5 + + + 100MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n + UTF-8 + + + + + true + + %d{yyy-MM-dd HH\:mm\:ss,SSS} %p %t - %m%n + UTF-8 + + + + + + + + + + + + + + + + + + + + + + + diff --git a/conf/logback_namesrv.xml b/conf/logback_namesrv.xml new file mode 100644 index 000000000..1d756f4f9 --- /dev/null +++ b/conf/logback_namesrv.xml @@ -0,0 +1,70 @@ + + + + ${user.home}/logs/rocketmqlogs/namesrv_default.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/namesrv_default.%i.log + + 1 + 5 + + + 100MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n + UTF-8 + + + + + ${user.home}/logs/rocketmqlogs/namesrv.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/namesrv.%i.log + + 1 + 5 + + + 100MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n + UTF-8 + + + + + true + + %d{yyy-MM-dd HH\:mm\:ss,SSS} %p %t - %m%n + UTF-8 + + + + + + + + + + + + + + + + + + + + + + + diff --git a/conf/logback_tools.xml b/conf/logback_tools.xml new file mode 100644 index 000000000..be337ed64 --- /dev/null +++ b/conf/logback_tools.xml @@ -0,0 +1,76 @@ + + + + ${user.home}/logs/rocketmqlogs/tools_default.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/tools_default.%i.log + + 1 + 5 + + + 100MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n + UTF-8 + + + + + ${user.home}/logs/rocketmqlogs/tools.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/tools.%i.log + + 1 + 5 + + + 100MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n + UTF-8 + + + + + + true + + %d{yyy-MM-dd HH\:mm\:ss,GMT+8} %p %t - %m%n + UTF-8 + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/deploy.bat b/deploy.bat new file mode 100644 index 000000000..b3334f031 --- /dev/null +++ b/deploy.bat @@ -0,0 +1 @@ +mvn -Dmaven.test.skip=true deploy \ No newline at end of file diff --git a/develop.sh b/develop.sh new file mode 100644 index 000000000..700b6f595 --- /dev/null +++ b/develop.sh @@ -0,0 +1,13 @@ +git pull + +rm -rf target +rm -f devenv + +if [ -z "$JAVA_HOME" ]; then + JAVA_HOME=/opt/taobao/java +fi + +export PATH=/opt/taobao/mvn/bin:$JAVA_HOME/bin:$PATH +mvn -Dmaven.test.skip=true clean package install assembly:assembly -U + +ln -s target/alibaba-rocketmq-3.2.0-SNAPSHOT.dir/alibaba-rocketmq devenv diff --git a/docs/rocketmq.java.code.style.xml b/docs/rocketmq.java.code.style.xml new file mode 100644 index 000000000..c0ac866d0 --- /dev/null +++ b/docs/rocketmq.java.code.style.xml @@ -0,0 +1,432 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/eclipse.bat b/eclipse.bat new file mode 100644 index 000000000..9b286a4c1 --- /dev/null +++ b/eclipse.bat @@ -0,0 +1 @@ +mvn -U eclipse:eclipse diff --git a/install.bat b/install.bat new file mode 100644 index 000000000..2c3b50576 --- /dev/null +++ b/install.bat @@ -0,0 +1,2 @@ +mvn -Dmaven.test.skip=true clean package install assembly:assembly -U + diff --git a/install.sh b/install.sh new file mode 100644 index 000000000..41e9f5efa --- /dev/null +++ b/install.sh @@ -0,0 +1,11 @@ +git pull + +rm -rf target +rm -f devenv +if [ -z "$JAVA_HOME" ]; then + JAVA_HOME=/opt/taobao/java +fi +export PATH=/opt/taobao/mvn/bin:$JAVA_HOME/bin:$PATH +mvn -Dmaven.test.skip=true clean package install assembly:assembly -U + +ln -s target/alibaba-rocketmq-3.2.0-SNAPSHOT.dir/alibaba-rocketmq devenv diff --git a/pom.xml b/pom.xml new file mode 100644 index 000000000..1ec5161fe --- /dev/null +++ b/pom.xml @@ -0,0 +1,365 @@ + + + + + + org.sonatype.oss + oss-parent + 7 + + + + + 4.0.0 + 2012 + com.alibaba.rocketmq + rocketmq-all + 3.2.2 + pom + rocketmq-all ${project.version} + https://github.com/alibaba/rocketmq + https://github.com/alibaba/RocketMQ/blob/develop/README.md + + + + rocketmq-client + rocketmq-common + rocketmq-broker + rocketmq-tools + rocketmq-store + rocketmq-namesrv + rocketmq-remoting + rocketmq-example + rocketmq-filtersrv + rocketmq-srvutil + + + + + vintagewang + https://github.com/vintagewang + vintage.wang@gmail.com + 8 + + + manhong + https://github.com/YangJodie + manhong.yqd@alibaba-inc.com + 8 + + + allenzhu + https://github.com/allenzhu + allen.jie.zhu@gmail.com + 8 + + + + + + Apache License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0 + + + + + http://gitlab.alibaba-inc.com/middleware/rocketmq.git + scm:git:http://gitlab.alibaba-inc.com/middleware/rocketmq.git + scm:git:http://gitlab.alibaba-inc.com/middleware/rocketmq.git + + + + UTF-8 + + true + true + true + + 1.6 + 1.6 + UTF-8 + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 2.3.2 + + ${java_source_version} + ${java_target_version} + ${file_encoding} + true + true + + + + org.apache.maven.plugins + maven-eclipse-plugin + 2.5.1 + + true + false + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.3 + + ${maven.test.skip} + -Xms512m -Xmx1024m + once + + **/*Test.java + + + com/alibaba/rocketmq/remoting/ExceptionTest.java + com/alibaba/rocketmq/remoting/SyncInvokeTest.java + com/alibaba/rocketmq/remoting/NettyIdleTest.java + com/alibaba/rocketmq/remoting/NettyConnectionTest.java + com/alibaba/rocketmq/common/filter/PolishExprTest.java + com/alibaba/rocketmq/common/protocol/MQProtosHelperTest.java + + com/alibaba/rocketmq/client/consumer/loadbalance/AllocateMessageQueueAveragelyTest.java + + com/alibaba/rocketmq/store/RecoverTest.java + com/alibaba/rocketmq/broker/api/SendMessageTest.java + com/alibaba/rocketmq/test/integration/*/*.java + com/alibaba/rocketmq/test/integration/BaseTest.java + com/alibaba/rocketmq/test/*/*.java + com/alibaba/rocketmq/test/BaseTest.java + + + + + + + maven-assembly-plugin + + alibaba-rocketmq-${project.version} + + release.xml + + + + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + 2.7 + + + attach-javadocs + + jar + + + + + ${maven.jdoc.skip} + ${file_encoding} + ${file_encoding} + org.jboss.apiviz.APIviz + + org.jboss.apiviz + apiviz + 1.3.0.GA + + true + true + true + true + true + + + + org.apache.maven.plugins + maven-source-plugin + 2.1.2 + + + attach-sources + + jar + + + + + + org.codehaus.mojo + clirr-maven-plugin + 2.6.1 + + + + + + src/main/resources + false + + + + + + + release-sign-artifacts + + + performRelease + true + + + + + + org.apache.maven.plugins + maven-gpg-plugin + 1.1 + + + sign-artifacts + verify + + sign + + + + + + + + + + + + + ${project.groupId} + rocketmq-client + ${project.version} + + + ${project.groupId} + rocketmq-broker + ${project.version} + + + ${project.groupId} + rocketmq-common + ${project.version} + + + ${project.groupId} + rocketmq-store + ${project.version} + + + ${project.groupId} + rocketmq-namesrv + ${project.version} + + + ${project.groupId} + rocketmq-tools + ${project.version} + + + ${project.groupId} + rocketmq-remoting + ${project.version} + + + ${project.groupId} + rocketmq-qatest + ${project.version} + + + ${project.groupId} + rocketmq-filtersrv + ${project.version} + + + ${project.groupId} + rocketmq-srvutil + ${project.version} + + + junit + junit + 4.11 + test + + + org.slf4j + slf4j-api + 1.7.5 + + + ch.qos.logback + logback-classic + 1.0.13 + + + ch.qos.logback + logback-core + 1.0.13 + + + commons-io + commons-io + 2.4 + + + commons-cli + commons-cli + 1.2 + + + io.netty + netty-all + 4.0.23.Final + + + com.alibaba + fastjson + 1.1.41 + + + mysql + mysql-connector-java + 5.1.31 + + + org.apache.derby + derby + 10.10.2.0 + + + jboss + javassist + 3.7.ga + + + + diff --git a/release-client.xml b/release-client.xml new file mode 100644 index 000000000..772140abc --- /dev/null +++ b/release-client.xml @@ -0,0 +1,33 @@ + + + alibaba-rocketmq-client-java + + dir + tar.gz + + + + + LICENSE.txt + + + + + + + + + com.alibaba.rocketmq:rocketmq-client + + + ./ + false + + + ./ + + + + + + diff --git a/release.xml b/release.xml new file mode 100644 index 000000000..ea51c9b33 --- /dev/null +++ b/release.xml @@ -0,0 +1,43 @@ + + + alibaba-rocketmq + + dir + tar.gz + + + + + bin/* + conf/* + conf/*/* + test/* + benchmark/* + LICENSE.txt + + + + + + + + + com.alibaba.rocketmq:rocketmq-broker + com.alibaba.rocketmq:rocketmq-tools + com.alibaba.rocketmq:rocketmq-client + com.alibaba.rocketmq:rocketmq-namesrv + com.alibaba.rocketmq:rocketmq-filtersrv + com.alibaba.rocketmq:rocketmq-example + + + lib/ + false + + + lib/ + + + + + + diff --git a/rocketmq-broker/pom.xml b/rocketmq-broker/pom.xml new file mode 100644 index 000000000..5013ece9b --- /dev/null +++ b/rocketmq-broker/pom.xml @@ -0,0 +1,65 @@ + + + com.alibaba.rocketmq + rocketmq-all + 3.2.2 + + + 4.0.0 + jar + rocketmq-broker + rocketmq-broker ${project.version} + + + + junit + junit + test + + + ${project.groupId} + rocketmq-common + + + ${project.groupId} + rocketmq-store + + + ${project.groupId} + rocketmq-remoting + + + ${project.groupId} + rocketmq-client + + + ${project.groupId} + rocketmq-srvutil + + + ch.qos.logback + logback-classic + + + ch.qos.logback + logback-core + + + com.alibaba + fastjson + + + mysql + mysql-connector-java + + + org.apache.derby + derby + + + jboss + javassist + + + diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/BrokerController.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/BrokerController.java new file mode 100644 index 000000000..8528a98f1 --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/BrokerController.java @@ -0,0 +1,837 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.broker; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.rocketmq.broker.client.ClientHousekeepingService; +import com.alibaba.rocketmq.broker.client.ConsumerIdsChangeListener; +import com.alibaba.rocketmq.broker.client.ConsumerManager; +import com.alibaba.rocketmq.broker.client.DefaultConsumerIdsChangeListener; +import com.alibaba.rocketmq.broker.client.ProducerManager; +import com.alibaba.rocketmq.broker.client.net.Broker2Client; +import com.alibaba.rocketmq.broker.client.rebalance.RebalanceLockManager; +import com.alibaba.rocketmq.broker.filtersrv.FilterServerManager; +import com.alibaba.rocketmq.broker.longpolling.PullRequestHoldService; +import com.alibaba.rocketmq.broker.mqtrace.ConsumeMessageHook; +import com.alibaba.rocketmq.broker.mqtrace.SendMessageHook; +import com.alibaba.rocketmq.broker.offset.ConsumerOffsetManager; +import com.alibaba.rocketmq.broker.out.BrokerOuterAPI; +import com.alibaba.rocketmq.broker.processor.AdminBrokerProcessor; +import com.alibaba.rocketmq.broker.processor.ClientManageProcessor; +import com.alibaba.rocketmq.broker.processor.EndTransactionProcessor; +import com.alibaba.rocketmq.broker.processor.PullMessageProcessor; +import com.alibaba.rocketmq.broker.processor.QueryMessageProcessor; +import com.alibaba.rocketmq.broker.processor.SendMessageProcessor; +import com.alibaba.rocketmq.broker.slave.SlaveSynchronize; +import com.alibaba.rocketmq.broker.subscription.SubscriptionGroupManager; +import com.alibaba.rocketmq.broker.topic.TopicConfigManager; +import com.alibaba.rocketmq.common.BrokerConfig; +import com.alibaba.rocketmq.common.DataVersion; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.ThreadFactoryImpl; +import com.alibaba.rocketmq.common.TopicConfig; +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.common.constant.PermName; +import com.alibaba.rocketmq.common.namesrv.RegisterBrokerResult; +import com.alibaba.rocketmq.common.protocol.RequestCode; +import com.alibaba.rocketmq.common.protocol.body.TopicConfigSerializeWrapper; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.remoting.RemotingServer; +import com.alibaba.rocketmq.remoting.netty.NettyClientConfig; +import com.alibaba.rocketmq.remoting.netty.NettyRemotingServer; +import com.alibaba.rocketmq.remoting.netty.NettyRequestProcessor; +import com.alibaba.rocketmq.remoting.netty.NettyServerConfig; +import com.alibaba.rocketmq.store.DefaultMessageStore; +import com.alibaba.rocketmq.store.MessageStore; +import com.alibaba.rocketmq.store.config.BrokerRole; +import com.alibaba.rocketmq.store.config.MessageStoreConfig; +import com.alibaba.rocketmq.store.stats.BrokerStats; +import com.alibaba.rocketmq.store.stats.BrokerStatsManager; + + +/** + * Broker各个服务控制器 + * + * @author shijia.wxr + * @since 2013-7-26 + */ +public class BrokerController { + private static final Logger log = LoggerFactory.getLogger(LoggerName.BrokerLoggerName); + // 服务器配置 + private final BrokerConfig brokerConfig; + // 通信层配置 + private final NettyServerConfig nettyServerConfig; + private final NettyClientConfig nettyClientConfig; + // 存储层配置 + private final MessageStoreConfig messageStoreConfig; + // 配置文件版本号 + private final DataVersion configDataVersion = new DataVersion(); + // 消费进度存储 + private final ConsumerOffsetManager consumerOffsetManager; + // Consumer连接、订阅关系管理 + private final ConsumerManager consumerManager; + // Producer连接管理 + private final ProducerManager producerManager; + // 检测所有客户端连接 + private final ClientHousekeepingService clientHousekeepingService; + private final PullMessageProcessor pullMessageProcessor; + private final PullRequestHoldService pullRequestHoldService; + // Broker主动调用Client + private final Broker2Client broker2Client; + // 订阅组配置管理 + private final SubscriptionGroupManager subscriptionGroupManager; + // 订阅组内成员发生变化，立刻通知所有成员 + private final ConsumerIdsChangeListener consumerIdsChangeListener; + // 管理队列的锁分配 + private final RebalanceLockManager rebalanceLockManager = new RebalanceLockManager(); + // Broker的通信层客户端 + private final BrokerOuterAPI brokerOuterAPI; + private final ScheduledExecutorService scheduledExecutorService = Executors + .newSingleThreadScheduledExecutor(new ThreadFactoryImpl("BrokerControllerScheduledThread")); + // Slave定期从Master同步信息 + private final SlaveSynchronize slaveSynchronize; + // 存储层对象 + private MessageStore messageStore; + // 通信层对象 + private RemotingServer remotingServer; + // Topic配置 + private TopicConfigManager topicConfigManager; + // 处理发送消息线程池 + private ExecutorService sendMessageExecutor; + // 处理拉取消息线程池 + private ExecutorService pullMessageExecutor; + // 处理管理Broker线程池 + private ExecutorService adminBrokerExecutor; + // 处理管理Client线程池 + private ExecutorService clientManageExecutor; + // 是否需要定期更新HA Master地址 + private boolean updateMasterHAServerAddrPeriodically = false; + + private BrokerStats brokerStats; + // 对消息写入进行流控 + private final BlockingQueue sendThreadPoolQueue; + + // 对消息读取进行流控 + private final BlockingQueue pullThreadPoolQueue; + + // FilterServer管理 + private final FilterServerManager filterServerManager; + + private final BrokerStatsManager brokerStatsManager; + + + public BrokerController(// + final BrokerConfig brokerConfig, // + final NettyServerConfig nettyServerConfig, // + final NettyClientConfig nettyClientConfig, // + final MessageStoreConfig messageStoreConfig // + ) { + this.brokerConfig = brokerConfig; + this.nettyServerConfig = nettyServerConfig; + this.nettyClientConfig = nettyClientConfig; + this.messageStoreConfig = messageStoreConfig; + this.consumerOffsetManager = new ConsumerOffsetManager(this); + this.topicConfigManager = new TopicConfigManager(this); + this.pullMessageProcessor = new PullMessageProcessor(this); + this.pullRequestHoldService = new PullRequestHoldService(this); + this.consumerIdsChangeListener = new DefaultConsumerIdsChangeListener(this); + this.consumerManager = new ConsumerManager(this.consumerIdsChangeListener); + this.producerManager = new ProducerManager(); + this.clientHousekeepingService = new ClientHousekeepingService(this); + this.broker2Client = new Broker2Client(this); + this.subscriptionGroupManager = new SubscriptionGroupManager(this); + this.brokerOuterAPI = new BrokerOuterAPI(nettyClientConfig); + this.filterServerManager = new FilterServerManager(this); + + if (this.brokerConfig.getNamesrvAddr() != null) { + this.brokerOuterAPI.updateNameServerAddressList(this.brokerConfig.getNamesrvAddr()); + log.info("user specfied name server address: {}", this.brokerConfig.getNamesrvAddr()); + } + + this.slaveSynchronize = new SlaveSynchronize(this); + + this.sendThreadPoolQueue = + new LinkedBlockingQueue(this.brokerConfig.getSendThreadPoolQueueCapacity()); + + this.pullThreadPoolQueue = + new LinkedBlockingQueue(this.brokerConfig.getPullThreadPoolQueueCapacity()); + + this.brokerStatsManager = new BrokerStatsManager(this.brokerConfig.getBrokerClusterName()); + } + + + public boolean initialize() { + boolean result = true; + + // 加载Topic配置 + result = result && this.topicConfigManager.load(); + + // 加载Consumer Offset + result = result && this.consumerOffsetManager.load(); + // 加载Consumer subscription + result = result && this.subscriptionGroupManager.load(); + + // 初始化存储层 + if (result) { + try { + this.messageStore = new DefaultMessageStore(this.messageStoreConfig, this.brokerStatsManager); + } + catch (IOException e) { + result = false; + e.printStackTrace(); + } + } + + // 加载本地消息数据 + result = result && this.messageStore.load(); + + if (result) { + // 初始化通信层 + this.remotingServer = + new NettyRemotingServer(this.nettyServerConfig, this.clientHousekeepingService); + + // 初始化线程池 + this.sendMessageExecutor = new ThreadPoolExecutor(// + this.brokerConfig.getSendMessageThreadPoolNums(),// + this.brokerConfig.getSendMessageThreadPoolNums(),// + 1000 * 60,// + TimeUnit.MILLISECONDS,// + this.sendThreadPoolQueue,// + new ThreadFactoryImpl("SendMessageThread_")); + + this.pullMessageExecutor = new ThreadPoolExecutor(// + this.brokerConfig.getPullMessageThreadPoolNums(),// + this.brokerConfig.getPullMessageThreadPoolNums(),// + 1000 * 60,// + TimeUnit.MILLISECONDS,// + this.pullThreadPoolQueue,// + new ThreadFactoryImpl("PullMessageThread_")); + + this.adminBrokerExecutor = + Executors.newFixedThreadPool(this.brokerConfig.getAdminBrokerThreadPoolNums(), + new ThreadFactoryImpl("AdminBrokerThread_")); + + this.clientManageExecutor = + Executors.newFixedThreadPool(this.brokerConfig.getClientManageThreadPoolNums(), + new ThreadFactoryImpl("ClientManageThread_")); + + this.registerProcessor(); + + this.brokerStats = new BrokerStats((DefaultMessageStore) this.messageStore); + + // 每天凌晨00:00:00统计消息量 + final long initialDelay = UtilAll.computNextMorningTimeMillis() - System.currentTimeMillis(); + final long period = 1000 * 60 * 60 * 24; + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + try { + BrokerController.this.getBrokerStats().record(); + } + catch (Exception e) { + log.error("schedule record error.", e); + } + } + }, initialDelay, period, TimeUnit.MILLISECONDS); + + // 定时刷消费进度 + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + try { + BrokerController.this.consumerOffsetManager.persist(); + } + catch (Exception e) { + log.error("schedule persist consumerOffset error.", e); + } + } + }, 1000 * 10, this.brokerConfig.getFlushConsumerOffsetInterval(), TimeUnit.MILLISECONDS); + + // 定时删除非常落后的消费进度，10分钟扫描一次 + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + try { + BrokerController.this.consumerOffsetManager.scanUnsubscribedTopic(); + } + catch (Exception e) { + log.error("schedule scanUnsubscribedTopic error.", e); + } + } + }, 10, 60, TimeUnit.MINUTES); + + // 先获取Name Server地址 + if (this.brokerConfig.getNamesrvAddr() != null) { + this.brokerOuterAPI.updateNameServerAddressList(this.brokerConfig.getNamesrvAddr()); + } + // 定时获取Name Server地址 + else if (this.brokerConfig.isFetchNamesrvAddrByAddressServer()) { + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + + @Override + public void run() { + try { + BrokerController.this.brokerOuterAPI.fetchNameServerAddr(); + } + catch (Exception e) { + log.error("ScheduledTask fetchNameServerAddr exception", e); + } + } + }, 1000 * 10, 1000 * 60 * 2, TimeUnit.MILLISECONDS); + } + + // 如果是slave + if (BrokerRole.SLAVE == this.messageStoreConfig.getBrokerRole()) { + if (this.messageStoreConfig.getHaMasterAddress() != null + && this.messageStoreConfig.getHaMasterAddress().length() >= 6) { + this.messageStore.updateHaMasterAddress(this.messageStoreConfig.getHaMasterAddress()); + this.updateMasterHAServerAddrPeriodically = false; + } + else { + this.updateMasterHAServerAddrPeriodically = true; + } + + // Slave定时从Master同步配置信息 + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + + @Override + public void run() { + try { + BrokerController.this.slaveSynchronize.syncAll(); + } + catch (Exception e) { + log.error("ScheduledTask syncAll slave exception", e); + } + } + }, 1000 * 10, 1000 * 60, TimeUnit.MILLISECONDS); + } + // 如果是Master，增加统计日志 + else { + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + + @Override + public void run() { + try { + BrokerController.this.printMasterAndSlaveDiff(); + } + catch (Exception e) { + log.error("schedule printMasterAndSlaveDiff error.", e); + } + } + }, 1000 * 10, 1000 * 60, TimeUnit.MILLISECONDS); + } + } + + return result; + } + + + public void registerProcessor() { + /** + * SendMessageProcessor + */ + SendMessageProcessor sendProcessor = new SendMessageProcessor(this); + sendProcessor.registerSendMessageHook(sendMessageHookList); + this.remotingServer.registerProcessor(RequestCode.SEND_MESSAGE, sendProcessor, + this.sendMessageExecutor); + this.remotingServer.registerProcessor(RequestCode.SEND_MESSAGE_V2, sendProcessor, + this.sendMessageExecutor); + this.remotingServer.registerProcessor(RequestCode.CONSUMER_SEND_MSG_BACK, sendProcessor, + this.sendMessageExecutor); + + /** + * PullMessageProcessor + */ + this.remotingServer.registerProcessor(RequestCode.PULL_MESSAGE, this.pullMessageProcessor, + this.pullMessageExecutor); + this.pullMessageProcessor.registerConsumeMessageHook(consumeMessageHookList); + + /** + * QueryMessageProcessor + */ + NettyRequestProcessor queryProcessor = new QueryMessageProcessor(this); + this.remotingServer.registerProcessor(RequestCode.QUERY_MESSAGE, queryProcessor, + this.pullMessageExecutor); + this.remotingServer.registerProcessor(RequestCode.VIEW_MESSAGE_BY_ID, queryProcessor, + this.pullMessageExecutor); + + /** + * ClientManageProcessor + */ + ClientManageProcessor clientProcessor = new ClientManageProcessor(this); + clientProcessor.registerConsumeMessageHook(this.consumeMessageHookList); + this.remotingServer.registerProcessor(RequestCode.HEART_BEAT, clientProcessor, + this.clientManageExecutor); + this.remotingServer.registerProcessor(RequestCode.UNREGISTER_CLIENT, clientProcessor, + this.clientManageExecutor); + this.remotingServer.registerProcessor(RequestCode.GET_CONSUMER_LIST_BY_GROUP, clientProcessor, + this.clientManageExecutor); + + /** + * Offset存储更新转移到ClientProcessor处理 + */ + this.remotingServer.registerProcessor(RequestCode.UPDATE_CONSUMER_OFFSET, clientProcessor, + this.clientManageExecutor); + this.remotingServer.registerProcessor(RequestCode.QUERY_CONSUMER_OFFSET, clientProcessor, + this.clientManageExecutor); + + /** + * EndTransactionProcessor + */ + this.remotingServer.registerProcessor(RequestCode.END_TRANSACTION, new EndTransactionProcessor(this), + this.sendMessageExecutor); + + /** + * Default + */ + AdminBrokerProcessor adminProcessor = new AdminBrokerProcessor(this); + this.remotingServer.registerDefaultProcessor(adminProcessor, this.adminBrokerExecutor); + } + + + public Broker2Client getBroker2Client() { + return broker2Client; + } + + + public BrokerConfig getBrokerConfig() { + return brokerConfig; + } + + + public String getConfigDataVersion() { + return this.configDataVersion.toJson(); + } + + + public ConsumerManager getConsumerManager() { + return consumerManager; + } + + + public ConsumerOffsetManager getConsumerOffsetManager() { + return consumerOffsetManager; + } + + + public MessageStore getMessageStore() { + return messageStore; + } + + + public void setMessageStore(MessageStore messageStore) { + this.messageStore = messageStore; + } + + + public MessageStoreConfig getMessageStoreConfig() { + return messageStoreConfig; + } + + + public NettyServerConfig getNettyServerConfig() { + return nettyServerConfig; + } + + + public ProducerManager getProducerManager() { + return producerManager; + } + + + public PullMessageProcessor getPullMessageProcessor() { + return pullMessageProcessor; + } + + + public PullRequestHoldService getPullRequestHoldService() { + return pullRequestHoldService; + } + + + public RemotingServer getRemotingServer() { + return remotingServer; + } + + + public void setRemotingServer(RemotingServer remotingServer) { + this.remotingServer = remotingServer; + } + + + public SubscriptionGroupManager getSubscriptionGroupManager() { + return subscriptionGroupManager; + } + + + public void shutdown() { + if (this.brokerStatsManager != null) { + this.brokerStatsManager.shutdown(); + } + + if (this.clientHousekeepingService != null) { + this.clientHousekeepingService.shutdown(); + } + + if (this.pullRequestHoldService != null) { + this.pullRequestHoldService.shutdown(); + } + + if (this.remotingServer != null) { + this.remotingServer.shutdown(); + } + + if (this.messageStore != null) { + this.messageStore.shutdown(); + } + + this.scheduledExecutorService.shutdown(); + try { + this.scheduledExecutorService.awaitTermination(5000, TimeUnit.MILLISECONDS); + } + catch (InterruptedException e) { + } + + this.unregisterBrokerAll(); + + if (this.sendMessageExecutor != null) { + this.sendMessageExecutor.shutdown(); + } + + if (this.pullMessageExecutor != null) { + this.pullMessageExecutor.shutdown(); + } + + if (this.adminBrokerExecutor != null) { + this.adminBrokerExecutor.shutdown(); + } + + if (this.brokerOuterAPI != null) { + this.brokerOuterAPI.shutdown(); + } + + this.consumerOffsetManager.persist(); + + if (this.filterServerManager != null) { + this.filterServerManager.shutdown(); + } + } + + + private void unregisterBrokerAll() { + this.brokerOuterAPI.unregisterBrokerAll(// + this.brokerConfig.getBrokerClusterName(), // + this.getBrokerAddr(), // + this.brokerConfig.getBrokerName(), // + this.brokerConfig.getBrokerId()); + } + + + public String getBrokerAddr() { + String addr = this.brokerConfig.getBrokerIP1() + ":" + this.nettyServerConfig.getListenPort(); + return addr; + } + + + public void start() throws Exception { + if (this.messageStore != null) { + this.messageStore.start(); + } + + if (this.remotingServer != null) { + this.remotingServer.start(); + } + + if (this.brokerOuterAPI != null) { + this.brokerOuterAPI.start(); + } + + if (this.pullRequestHoldService != null) { + this.pullRequestHoldService.start(); + } + + if (this.clientHousekeepingService != null) { + this.clientHousekeepingService.start(); + } + + if (this.filterServerManager != null) { + this.filterServerManager.start(); + } + + // 启动时，强制注册 + this.registerBrokerAll(true); + + // 定时注册Broker到Name Server + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + + @Override + public void run() { + try { + BrokerController.this.registerBrokerAll(true); + } + catch (Exception e) { + log.error("registerBrokerAll Exception", e); + } + } + }, 1000 * 10, 1000 * 30, TimeUnit.MILLISECONDS); + + if (this.brokerStatsManager != null) { + this.brokerStatsManager.start(); + } + + // 删除多余的Topic + this.addDeleteTopicTask(); + } + + + public synchronized void registerBrokerAll(final boolean checkOrderConfig) { + TopicConfigSerializeWrapper topicConfigWrapper = + this.getTopicConfigManager().buildTopicConfigSerializeWrapper(); + + // 同步 Broker 读写权限 + if (!PermName.isWriteable(this.getBrokerConfig().getBrokerPermission()) + || !PermName.isReadable(this.getBrokerConfig().getBrokerPermission())) { + ConcurrentHashMap topicConfigTable = + new ConcurrentHashMap(topicConfigWrapper.getTopicConfigTable()); + for (TopicConfig topicConfig : topicConfigTable.values()) { + topicConfig.setPerm(this.getBrokerConfig().getBrokerPermission()); + } + topicConfigWrapper.setTopicConfigTable(topicConfigTable); + } + + RegisterBrokerResult registerBrokerResult = this.brokerOuterAPI.registerBrokerAll(// + this.brokerConfig.getBrokerClusterName(), // + this.getBrokerAddr(), // + this.brokerConfig.getBrokerName(), // + this.brokerConfig.getBrokerId(), // + this.getHAServerAddr(), // + topicConfigWrapper,// + this.filterServerManager.buildNewFilterServerList()// + ); + + if (registerBrokerResult != null) { + if (this.updateMasterHAServerAddrPeriodically && registerBrokerResult.getHaServerAddr() != null) { + this.messageStore.updateHaMasterAddress(registerBrokerResult.getHaServerAddr()); + } + + this.slaveSynchronize.setMasterAddr(registerBrokerResult.getMasterAddr()); + + // 检查 topic config 的顺序消息配置 + if (checkOrderConfig) { + this.getTopicConfigManager().updateOrderTopicConfig(registerBrokerResult.getKvTable()); + } + } + } + + + public TopicConfigManager getTopicConfigManager() { + return topicConfigManager; + } + + + public void setTopicConfigManager(TopicConfigManager topicConfigManager) { + this.topicConfigManager = topicConfigManager; + } + + + public String getHAServerAddr() { + String addr = this.brokerConfig.getBrokerIP2() + ":" + this.messageStoreConfig.getHaListenPort(); + return addr; + } + + + public void updateAllConfig(Properties properties) { + MixAll.properties2Object(properties, brokerConfig); + MixAll.properties2Object(properties, nettyServerConfig); + MixAll.properties2Object(properties, nettyClientConfig); + MixAll.properties2Object(properties, messageStoreConfig); + this.configDataVersion.nextVersion(); + this.flushAllConfig(); + } + + + private void flushAllConfig() { + String allConfig = this.encodeAllConfig(); + try { + MixAll.string2File(allConfig, BrokerPathConfigHelper.getBrokerConfigPath()); + log.info("flush broker config, {} OK", BrokerPathConfigHelper.getBrokerConfigPath()); + } + catch (IOException e) { + log.info("flush broker config Exception, " + BrokerPathConfigHelper.getBrokerConfigPath(), e); + } + } + + + public String encodeAllConfig() { + StringBuilder sb = new StringBuilder(); + { + Properties properties = MixAll.object2Properties(this.brokerConfig); + if (properties != null) { + sb.append(MixAll.properties2String(properties)); + } + else { + log.error("encodeAllConfig object2Properties error"); + } + } + + { + Properties properties = MixAll.object2Properties(this.messageStoreConfig); + if (properties != null) { + sb.append(MixAll.properties2String(properties)); + } + else { + log.error("encodeAllConfig object2Properties error"); + } + } + + { + Properties properties = MixAll.object2Properties(this.nettyServerConfig); + if (properties != null) { + sb.append(MixAll.properties2String(properties)); + } + else { + log.error("encodeAllConfig object2Properties error"); + } + } + + { + Properties properties = MixAll.object2Properties(this.nettyClientConfig); + if (properties != null) { + sb.append(MixAll.properties2String(properties)); + } + else { + log.error("encodeAllConfig object2Properties error"); + } + } + return sb.toString(); + } + + + public RebalanceLockManager getRebalanceLockManager() { + return rebalanceLockManager; + } + + + public SlaveSynchronize getSlaveSynchronize() { + return slaveSynchronize; + } + + + public BrokerOuterAPI getBrokerOuterAPI() { + return brokerOuterAPI; + } + + + public ExecutorService getPullMessageExecutor() { + return pullMessageExecutor; + } + + + public void setPullMessageExecutor(ExecutorService pullMessageExecutor) { + this.pullMessageExecutor = pullMessageExecutor; + } + + + public BrokerStats getBrokerStats() { + return brokerStats; + } + + + public void setBrokerStats(BrokerStats brokerStats) { + this.brokerStats = brokerStats; + } + + + public BlockingQueue getSendThreadPoolQueue() { + return sendThreadPoolQueue; + } + + + public FilterServerManager getFilterServerManager() { + return filterServerManager; + } + + + public BrokerStatsManager getBrokerStatsManager() { + return brokerStatsManager; + } + + + private void printMasterAndSlaveDiff() { + long diff = this.messageStore.slaveFallBehindMuch(); + + // XXX: warn and notify me + log.info("slave fall behind master, how much, {} bytes", diff); + } + + + public void addDeleteTopicTask() { + // 5分钟后，尝试删除topic + this.scheduledExecutorService.schedule(new Runnable() { + @Override + public void run() { + int removedTopicCnt = + BrokerController.this.messageStore.cleanUnusedTopic(BrokerController.this + .getTopicConfigManager().getTopicConfigTable().keySet()); + log.info("addDeleteTopicTask removed topic count {}", removedTopicCnt); + } + }, 5, TimeUnit.MINUTES); + } + + // 注册发送消息轨迹 hook + private final List sendMessageHookList = new ArrayList(); + + + public void registerSendMessageHook(final SendMessageHook hook) { + this.sendMessageHookList.add(hook); + log.info("register SendMessageHook Hook, {}", hook.hookName()); + } + + // 注册消费消息轨迹 hook + private final List consumeMessageHookList = new ArrayList(); + + + public void registerConsumeMessageHook(final ConsumeMessageHook hook) { + this.consumeMessageHookList.add(hook); + log.info("register ConsumeMessageHook Hook, {}", hook.hookName()); + } + + + public void registerServerRPCHook(RPCHook rpcHook) { + getRemotingServer().registerRPCHook(rpcHook); + } + + + public void registerClientRPCHook(RPCHook rpcHook) { + this.getBrokerOuterAPI().registerRPCHook(rpcHook); + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/BrokerPathConfigHelper.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/BrokerPathConfigHelper.java new file mode 100644 index 000000000..fd4862fcb --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/BrokerPathConfigHelper.java @@ -0,0 +1,35 @@ +package com.alibaba.rocketmq.broker; + +import java.io.File; + + +public class BrokerPathConfigHelper { + private static String brokerConfigPath = System.getProperty("user.home") + File.separator + "store" + + File.separator + "config" + File.separator + "broker.properties"; + + + public static String getBrokerConfigPath() { + return brokerConfigPath; + } + + + public static void setBrokerConfigPath(String path) { + brokerConfigPath = path; + } + + + public static String getTopicConfigPath(final String rootDir) { + return rootDir + File.separator + "config" + File.separator + "topics.json"; + } + + + public static String getConsumerOffsetPath(final String rootDir) { + return rootDir + File.separator + "config" + File.separator + "consumerOffset.json"; + } + + + public static String getSubscriptionGroupPath(final String rootDir) { + return rootDir + File.separator + "config" + File.separator + "subscriptionGroup.json"; + } + +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/BrokerStartup.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/BrokerStartup.java new file mode 100644 index 000000000..d06359ba1 --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/BrokerStartup.java @@ -0,0 +1,285 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.broker; + +import java.io.BufferedInputStream; +import java.io.FileInputStream; +import java.io.InputStream; +import java.util.Properties; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.PosixParser; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import ch.qos.logback.classic.LoggerContext; +import ch.qos.logback.classic.joran.JoranConfigurator; + +import com.alibaba.rocketmq.common.BrokerConfig; +import com.alibaba.rocketmq.common.MQVersion; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.remoting.common.RemotingUtil; +import com.alibaba.rocketmq.remoting.netty.NettyClientConfig; +import com.alibaba.rocketmq.remoting.netty.NettyServerConfig; +import com.alibaba.rocketmq.remoting.netty.NettySystemConfig; +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; +import com.alibaba.rocketmq.srvutil.ServerUtil; +import com.alibaba.rocketmq.store.config.BrokerRole; +import com.alibaba.rocketmq.store.config.MessageStoreConfig; + + +/** + * Broker启动入口 + * + * @author shijia.wxr + * @since 2013-7-26 + */ +public class BrokerStartup { + public static Properties properties = null; + public static CommandLine commandLine = null; + public static String configFile = null; + public static Logger log; + + + public static Options buildCommandlineOptions(final Options options) { + Option opt = new Option("c", "configFile", true, "Broker config properties file"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("p", "printConfigItem", false, "Print all config item"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("m", "printImportantConfig", false, "Print important config item"); + opt.setRequired(false); + options.addOption(opt); + + return options; + } + + + public static void main(String[] args) { + start(createBrokerController(args)); + } + + + public static BrokerController createBrokerController(String[] args) { + System.setProperty(RemotingCommand.RemotingVersionKey, Integer.toString(MQVersion.CurrentVersion)); + + // Socket发送缓冲区大小 + if (null == System.getProperty(NettySystemConfig.SystemPropertySocketSndbufSize)) { + NettySystemConfig.SocketSndbufSize = 131072; + } + + // Socket接收缓冲区大小 + if (null == System.getProperty(NettySystemConfig.SystemPropertySocketRcvbufSize)) { + NettySystemConfig.SocketRcvbufSize = 131072; + } + + try { + // 解析命令行 + Options options = ServerUtil.buildCommandlineOptions(new Options()); + commandLine = + ServerUtil.parseCmdLine("mqbroker", args, buildCommandlineOptions(options), + new PosixParser()); + if (null == commandLine) { + System.exit(-1); + return null; + } + + // 初始化配置文件 + final BrokerConfig brokerConfig = new BrokerConfig(); + final NettyServerConfig nettyServerConfig = new NettyServerConfig(); + final NettyClientConfig nettyClientConfig = new NettyClientConfig(); + nettyServerConfig.setListenPort(10911); + final MessageStoreConfig messageStoreConfig = new MessageStoreConfig(); + + // 如果是slave，修改默认值 + if (BrokerRole.SLAVE == messageStoreConfig.getBrokerRole()) { + int ratio = messageStoreConfig.getAccessMessageInMemoryMaxRatio() - 10; + messageStoreConfig.setAccessMessageInMemoryMaxRatio(ratio); + } + + // 打印默认配置 + if (commandLine.hasOption('p')) { + MixAll.printObjectProperties(null, brokerConfig); + MixAll.printObjectProperties(null, nettyServerConfig); + MixAll.printObjectProperties(null, nettyClientConfig); + MixAll.printObjectProperties(null, messageStoreConfig); + System.exit(0); + } + else if (commandLine.hasOption('m')) { + MixAll.printObjectProperties(null, brokerConfig, true); + MixAll.printObjectProperties(null, nettyServerConfig, true); + MixAll.printObjectProperties(null, nettyClientConfig, true); + MixAll.printObjectProperties(null, messageStoreConfig, true); + System.exit(0); + } + + // 指定配置文件 + if (commandLine.hasOption('c')) { + String file = commandLine.getOptionValue('c'); + if (file != null) { + configFile = file; + InputStream in = new BufferedInputStream(new FileInputStream(file)); + properties = new Properties(); + properties.load(in); + MixAll.properties2Object(properties, brokerConfig); + MixAll.properties2Object(properties, nettyServerConfig); + MixAll.properties2Object(properties, nettyClientConfig); + MixAll.properties2Object(properties, messageStoreConfig); + + BrokerPathConfigHelper.setBrokerConfigPath(file); + + System.out.println("load config properties file OK, " + file); + } + } + + MixAll.properties2Object(ServerUtil.commandLine2Properties(commandLine), brokerConfig); + + if (null == brokerConfig.getRocketmqHome()) { + System.out.println("Please set the " + MixAll.ROCKETMQ_HOME_ENV + + " variable in your environment to match the location of the RocketMQ installation"); + System.exit(-2); + } + + // 检测Name Server地址设置是否正确 IP:PORT + String namesrvAddr = brokerConfig.getNamesrvAddr(); + if (null != namesrvAddr) { + try { + String[] addrArray = namesrvAddr.split(";"); + if (addrArray != null) { + for (String addr : addrArray) { + RemotingUtil.string2SocketAddress(addr); + } + } + } + catch (Exception e) { + System.out + .printf( + "The Name Server Address[%s] illegal, please set it as follows, \"127.0.0.1:9876;192.168.0.1:9876\"\n", + namesrvAddr); + System.exit(-3); + } + } + + // BrokerId的处理 + switch (messageStoreConfig.getBrokerRole()) { + case ASYNC_MASTER: + case SYNC_MASTER: + // Master Id必须是0 + brokerConfig.setBrokerId(MixAll.MASTER_ID); + break; + case SLAVE: + if (brokerConfig.getBrokerId() <= 0) { + System.out.println("Slave's brokerId must be > 0"); + System.exit(-3); + } + + break; + default: + break; + } + + // Master监听Slave请求的端口，默认为服务端口+1 + messageStoreConfig.setHaListenPort(nettyServerConfig.getListenPort() + 1); + + // 初始化Logback + LoggerContext lc = (LoggerContext) LoggerFactory.getILoggerFactory(); + JoranConfigurator configurator = new JoranConfigurator(); + configurator.setContext(lc); + lc.reset(); + configurator.doConfigure(brokerConfig.getRocketmqHome() + "/conf/logback_broker.xml"); + log = LoggerFactory.getLogger(LoggerName.BrokerLoggerName); + + // 打印启动参数 + MixAll.printObjectProperties(log, brokerConfig); + MixAll.printObjectProperties(log, nettyServerConfig); + MixAll.printObjectProperties(log, nettyClientConfig); + MixAll.printObjectProperties(log, messageStoreConfig); + + // 初始化服务控制对象 + final BrokerController controller = new BrokerController(// + brokerConfig, // + nettyServerConfig, // + nettyClientConfig, // + messageStoreConfig); + boolean initResult = controller.initialize(); + if (!initResult) { + controller.shutdown(); + System.exit(-3); + } + + Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() { + private volatile boolean hasShutdown = false; + private AtomicInteger shutdownTimes = new AtomicInteger(0); + + + @Override + public void run() { + synchronized (this) { + log.info("shutdown hook was invoked, " + this.shutdownTimes.incrementAndGet()); + if (!this.hasShutdown) { + this.hasShutdown = true; + long begineTime = System.currentTimeMillis(); + controller.shutdown(); + long consumingTimeTotal = System.currentTimeMillis() - begineTime; + log.info("shutdown hook over, consuming time total(ms): " + consumingTimeTotal); + } + } + } + }, "ShutdownHook")); + + return controller; + } + catch (Throwable e) { + e.printStackTrace(); + System.exit(-1); + } + + return null; + } + + + public static BrokerController start(BrokerController controller) { + try { + // 启动服务控制对象 + controller.start(); + String tip = + "The broker[" + controller.getBrokerConfig().getBrokerName() + ", " + + controller.getBrokerAddr() + "] boot success."; + + if (null != controller.getBrokerConfig().getNamesrvAddr()) { + tip += " and name server is " + controller.getBrokerConfig().getNamesrvAddr(); + } + + log.info(tip); + System.out.println(tip); + + return controller; + } + catch (Throwable e) { + e.printStackTrace(); + System.exit(-1); + } + + return null; + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/client/ClientChannelInfo.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/client/ClientChannelInfo.java new file mode 100644 index 000000000..55811f65b --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/client/ClientChannelInfo.java @@ -0,0 +1,117 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.broker.client; + +import io.netty.channel.Channel; + +import com.alibaba.rocketmq.remoting.protocol.LanguageCode; + + +/** + * @author shijia.wxr + * @since 2013-7-26 + */ +public class ClientChannelInfo { + private final Channel channel; + private final String clientId; + private final LanguageCode language; + private final int version; + private volatile long lastUpdateTimestamp = System.currentTimeMillis(); + + + public ClientChannelInfo(Channel channel) { + this(channel, null, null, 0); + } + + + public ClientChannelInfo(Channel channel, String clientId, LanguageCode language, int version) { + this.channel = channel; + this.clientId = clientId; + this.language = language; + this.version = version; + } + + + public Channel getChannel() { + return channel; + } + + + public String getClientId() { + return clientId; + } + + + public LanguageCode getLanguage() { + return language; + } + + + public int getVersion() { + return version; + } + + + public long getLastUpdateTimestamp() { + return lastUpdateTimestamp; + } + + + public void setLastUpdateTimestamp(long lastUpdateTimestamp) { + this.lastUpdateTimestamp = lastUpdateTimestamp; + } + + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((channel == null) ? 0 : channel.hashCode()); + result = prime * result + ((clientId == null) ? 0 : clientId.hashCode()); + result = prime * result + ((language == null) ? 0 : language.hashCode()); + result = prime * result + (int) (lastUpdateTimestamp ^ (lastUpdateTimestamp >>> 32)); + result = prime * result + version; + return result; + } + + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + ClientChannelInfo other = (ClientChannelInfo) obj; + if (channel == null) { + if (other.channel != null) + return false; + } + else if (this.channel != other.channel) { + return false; + } + + return true; + } + + + @Override + public String toString() { + return "ClientChannelInfo [channel=" + channel + ", clientId=" + clientId + ", language=" + language + + ", version=" + version + ", lastUpdateTimestamp=" + lastUpdateTimestamp + "]"; + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/client/ClientHousekeepingService.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/client/ClientHousekeepingService.java new file mode 100644 index 000000000..8f5468b5d --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/client/ClientHousekeepingService.java @@ -0,0 +1,108 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.broker.client; + +import io.netty.channel.Channel; + +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.rocketmq.broker.BrokerController; +import com.alibaba.rocketmq.common.ThreadFactoryImpl; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.remoting.ChannelEventListener; + + +/** + * 定期检测客户端连接，清除不活动的连接 + * + * @author shijia.wxr + * @since 2013-7-26 + */ +public class ClientHousekeepingService implements ChannelEventListener { + private static final Logger log = LoggerFactory.getLogger(LoggerName.BrokerLoggerName); + private final BrokerController brokerController; + + private ScheduledExecutorService scheduledExecutorService = Executors + .newSingleThreadScheduledExecutor(new ThreadFactoryImpl("ClientHousekeepingScheduledThread")); + + + public ClientHousekeepingService(final BrokerController brokerController) { + this.brokerController = brokerController; + } + + + public void start() { + // 定时扫描过期的连接 + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + try { + ClientHousekeepingService.this.scanExceptionChannel(); + } + catch (Exception e) { + log.error("", e); + } + } + }, 1000 * 10, 1000 * 10, TimeUnit.MILLISECONDS); + } + + + public void shutdown() { + this.scheduledExecutorService.shutdown(); + } + + + private void scanExceptionChannel() { + this.brokerController.getProducerManager().scanNotActiveChannel(); + this.brokerController.getConsumerManager().scanNotActiveChannel(); + this.brokerController.getFilterServerManager().scanNotActiveChannel(); + } + + + @Override + public void onChannelConnect(String remoteAddr, Channel channel) { + + } + + + @Override + public void onChannelClose(String remoteAddr, Channel channel) { + this.brokerController.getProducerManager().doChannelCloseEvent(remoteAddr, channel); + this.brokerController.getConsumerManager().doChannelCloseEvent(remoteAddr, channel); + this.brokerController.getFilterServerManager().doChannelCloseEvent(remoteAddr, channel); + } + + + @Override + public void onChannelException(String remoteAddr, Channel channel) { + this.brokerController.getProducerManager().doChannelCloseEvent(remoteAddr, channel); + this.brokerController.getConsumerManager().doChannelCloseEvent(remoteAddr, channel); + this.brokerController.getFilterServerManager().doChannelCloseEvent(remoteAddr, channel); + } + + + @Override + public void onChannelIdle(String remoteAddr, Channel channel) { + this.brokerController.getProducerManager().doChannelCloseEvent(remoteAddr, channel); + this.brokerController.getConsumerManager().doChannelCloseEvent(remoteAddr, channel); + this.brokerController.getFilterServerManager().doChannelCloseEvent(remoteAddr, channel); + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/client/ConsumerGroupInfo.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/client/ConsumerGroupInfo.java new file mode 100644 index 000000000..e262be6d7 --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/client/ConsumerGroupInfo.java @@ -0,0 +1,281 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.broker.client; + +import io.netty.channel.Channel; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map.Entry; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.common.consumer.ConsumeFromWhere; +import com.alibaba.rocketmq.common.protocol.heartbeat.ConsumeType; +import com.alibaba.rocketmq.common.protocol.heartbeat.MessageModel; +import com.alibaba.rocketmq.common.protocol.heartbeat.SubscriptionData; + + +/** + * 整个Consumer Group信息 + * + * @author shijia.wxr + * @since 2013-7-26 + */ +public class ConsumerGroupInfo { + private static final Logger log = LoggerFactory.getLogger(LoggerName.BrokerLoggerName); + private final String groupName; + private final ConcurrentHashMap subscriptionTable = + new ConcurrentHashMap(); + private final ConcurrentHashMap channelInfoTable = + new ConcurrentHashMap(16); + private volatile ConsumeType consumeType; + private volatile MessageModel messageModel; + private volatile ConsumeFromWhere consumeFromWhere; + private volatile long lastUpdateTimestamp = System.currentTimeMillis(); + + + public ConsumerGroupInfo(String groupName, ConsumeType consumeType, MessageModel messageModel, + ConsumeFromWhere consumeFromWhere) { + this.groupName = groupName; + this.consumeType = consumeType; + this.messageModel = messageModel; + this.consumeFromWhere = consumeFromWhere; + } + + + public ClientChannelInfo findChannel(final String clientId) { + Iterator> it = this.channelInfoTable.entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + if (next.getValue().getClientId().equals(clientId)) { + return next.getValue(); + } + } + + return null; + } + + + public ConcurrentHashMap getSubscriptionTable() { + return subscriptionTable; + } + + + public ConcurrentHashMap getChannelInfoTable() { + return channelInfoTable; + } + + + public List getAllChannel() { + List result = new ArrayList(); + + result.addAll(this.channelInfoTable.keySet()); + + return result; + } + + + public List getAllClientId() { + List result = new ArrayList(); + + Iterator> it = this.channelInfoTable.entrySet().iterator(); + + while (it.hasNext()) { + Entry entry = it.next(); + ClientChannelInfo clientChannelInfo = entry.getValue(); + result.add(clientChannelInfo.getClientId()); + } + + return result; + } + + + public void unregisterChannel(final ClientChannelInfo clientChannelInfo) { + ClientChannelInfo old = this.channelInfoTable.remove(clientChannelInfo.getChannel()); + if (old != null) { + log.info("unregister a consumer[{}] from consumerGroupInfo {}", this.groupName, old.toString()); + } + } + + + public boolean doChannelCloseEvent(final String remoteAddr, final Channel channel) { + final ClientChannelInfo info = this.channelInfoTable.remove(channel); + if (info != null) { + log.warn( + "NETTY EVENT: remove not active channel[{}] from ConsumerGroupInfo groupChannelTable, consumer group: {}", + info.toString(), groupName); + return true; + } + + return false; + } + + + /** + * 返回值表示是否发生变更 + */ + public boolean updateChannel(final ClientChannelInfo infoNew, ConsumeType consumeType, + MessageModel messageModel, ConsumeFromWhere consumeFromWhere) { + boolean updated = false; + this.consumeType = consumeType; + this.messageModel = messageModel; + this.consumeFromWhere = consumeFromWhere; + + ClientChannelInfo infoOld = this.channelInfoTable.get(infoNew.getChannel()); + if (null == infoOld) { + ClientChannelInfo prev = this.channelInfoTable.put(infoNew.getChannel(), infoNew); + if (null == prev) { + log.info("new consumer connected, group: {} {} {} channel: {}", this.groupName, consumeType, + messageModel, infoNew.toString()); + updated = true; + } + + infoOld = infoNew; + } + else { + if (!infoOld.getClientId().equals(infoNew.getClientId())) { + log.error( + "[BUG] consumer channel exist in broker, but clientId not equal. GROUP: {} OLD: {} NEW: {} ", + this.groupName,// + infoOld.toString(),// + infoNew.toString()); + this.channelInfoTable.put(infoNew.getChannel(), infoNew); + } + } + + this.lastUpdateTimestamp = System.currentTimeMillis(); + infoOld.setLastUpdateTimestamp(this.lastUpdateTimestamp); + + return updated; + } + + + /** + * 返回值表示是否发生变更 + */ + public boolean updateSubscription(final Set subList) { + boolean updated = false; + // 增加新的订阅关系 + for (SubscriptionData sub : subList) { + SubscriptionData old = this.subscriptionTable.get(sub.getTopic()); + if (old == null) { + SubscriptionData prev = this.subscriptionTable.put(sub.getTopic(), sub); + if (null == prev) { + updated = true; + log.info("subscription changed, add new topic, group: {} {}", this.groupName, + sub.toString()); + } + } + else if (sub.getSubVersion() > old.getSubVersion()) { + if (this.consumeType == ConsumeType.CONSUME_PASSIVELY) { + log.info("subscription changed, group: {} OLD: {} NEW: {}", // + this.groupName,// + old.toString(),// + sub.toString()// + ); + } + + this.subscriptionTable.put(sub.getTopic(), sub); + } + } + + // 删除老的订阅关系 + Iterator> it = this.subscriptionTable.entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + String oldTopic = next.getKey(); + + boolean exist = false; + for (SubscriptionData sub : subList) { + if (sub.getTopic().equals(oldTopic)) { + exist = true; + break; + } + } + + if (!exist) { + log.warn("subscription changed, group: {} remove topic {} {}", // + this.groupName,// + oldTopic,// + next.getValue().toString()// + ); + + it.remove(); + updated = true; + } + } + + this.lastUpdateTimestamp = System.currentTimeMillis(); + + return updated; + } + + + public SubscriptionData findSubscriptionData(final String topic) { + return this.subscriptionTable.get(topic); + } + + + public ConsumeType getConsumeType() { + return consumeType; + } + + + public void setConsumeType(ConsumeType consumeType) { + this.consumeType = consumeType; + } + + + public MessageModel getMessageModel() { + return messageModel; + } + + + public void setMessageModel(MessageModel messageModel) { + this.messageModel = messageModel; + } + + + public String getGroupName() { + return groupName; + } + + + public long getLastUpdateTimestamp() { + return lastUpdateTimestamp; + } + + + public void setLastUpdateTimestamp(long lastUpdateTimestamp) { + this.lastUpdateTimestamp = lastUpdateTimestamp; + } + + + public ConsumeFromWhere getConsumeFromWhere() { + return consumeFromWhere; + } + + + public void setConsumeFromWhere(ConsumeFromWhere consumeFromWhere) { + this.consumeFromWhere = consumeFromWhere; + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/client/ConsumerIdsChangeListener.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/client/ConsumerIdsChangeListener.java new file mode 100644 index 000000000..65a2935af --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/client/ConsumerIdsChangeListener.java @@ -0,0 +1,29 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.broker.client; + +import io.netty.channel.Channel; + +import java.util.List; + + +/** + * @author shijia.wxr + * @since 2013-6-24 + */ +public interface ConsumerIdsChangeListener { + public void consumerIdsChanged(final String group, final List channels); +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/client/ConsumerManager.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/client/ConsumerManager.java new file mode 100644 index 000000000..b01d8f93e --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/client/ConsumerManager.java @@ -0,0 +1,202 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.broker.client; + +import io.netty.channel.Channel; + +import java.util.HashSet; +import java.util.Iterator; +import java.util.Map.Entry; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.common.consumer.ConsumeFromWhere; +import com.alibaba.rocketmq.common.protocol.heartbeat.ConsumeType; +import com.alibaba.rocketmq.common.protocol.heartbeat.MessageModel; +import com.alibaba.rocketmq.common.protocol.heartbeat.SubscriptionData; +import com.alibaba.rocketmq.remoting.common.RemotingHelper; +import com.alibaba.rocketmq.remoting.common.RemotingUtil; + + +/** + * Consumer连接、订阅关系管理 + * + * @author shijia.wxr + * @since 2013-7-26 + */ +public class ConsumerManager { + private static final Logger log = LoggerFactory.getLogger(LoggerName.BrokerLoggerName); + private final ConcurrentHashMap consumerTable = + new ConcurrentHashMap(1024); + + private final ConsumerIdsChangeListener consumerIdsChangeListener; + private static final long ChannelExpiredTimeout = 1000 * 120; + + + public ConsumerManager(final ConsumerIdsChangeListener consumerIdsChangeListener) { + this.consumerIdsChangeListener = consumerIdsChangeListener; + } + + + public ClientChannelInfo findChannel(final String group, final String clientId) { + ConsumerGroupInfo consumerGroupInfo = this.consumerTable.get(group); + if (consumerGroupInfo != null) { + return consumerGroupInfo.findChannel(clientId); + } + + return null; + } + + + public ConsumerGroupInfo getConsumerGroupInfo(final String group) { + return this.consumerTable.get(group); + } + + + public SubscriptionData findSubscriptionData(final String group, final String topic) { + ConsumerGroupInfo consumerGroupInfo = this.getConsumerGroupInfo(group); + if (consumerGroupInfo != null) { + return consumerGroupInfo.findSubscriptionData(topic); + } + + return null; + } + + + public int findSubscriptionDataCount(final String group) { + ConsumerGroupInfo consumerGroupInfo = this.getConsumerGroupInfo(group); + if (consumerGroupInfo != null) { + return consumerGroupInfo.getSubscriptionTable().size(); + } + + return 0; + } + + + public void doChannelCloseEvent(final String remoteAddr, final Channel channel) { + Iterator> it = this.consumerTable.entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + ConsumerGroupInfo info = next.getValue(); + boolean removed = info.doChannelCloseEvent(remoteAddr, channel); + if (removed) { + if (info.getChannelInfoTable().isEmpty()) { + ConsumerGroupInfo remove = this.consumerTable.remove(next.getKey()); + if (remove != null) { + log.info("ungister consumer ok, no any connection, and remove consumer group, {}", + next.getKey()); + } + } + + this.consumerIdsChangeListener.consumerIdsChanged(next.getKey(), info.getAllChannel()); + } + } + } + + + /** + * 返回是否有变化 + */ + public boolean registerConsumer(final String group, final ClientChannelInfo clientChannelInfo, + ConsumeType consumeType, MessageModel messageModel, ConsumeFromWhere consumeFromWhere, + final Set subList) { + ConsumerGroupInfo consumerGroupInfo = this.consumerTable.get(group); + if (null == consumerGroupInfo) { + ConsumerGroupInfo tmp = new ConsumerGroupInfo(group, consumeType, messageModel, consumeFromWhere); + ConsumerGroupInfo prev = this.consumerTable.putIfAbsent(group, tmp); + consumerGroupInfo = prev != null ? prev : tmp; + } + + boolean r1 = + consumerGroupInfo.updateChannel(clientChannelInfo, consumeType, messageModel, + consumeFromWhere); + boolean r2 = consumerGroupInfo.updateSubscription(subList); + + if (r1 || r2) { + this.consumerIdsChangeListener.consumerIdsChanged(group, consumerGroupInfo.getAllChannel()); + } + + return r1 || r2; + } + + + public void unregisterConsumer(final String group, final ClientChannelInfo clientChannelInfo) { + ConsumerGroupInfo consumerGroupInfo = this.consumerTable.get(group); + if (null != consumerGroupInfo) { + consumerGroupInfo.unregisterChannel(clientChannelInfo); + if (consumerGroupInfo.getChannelInfoTable().isEmpty()) { + ConsumerGroupInfo remove = this.consumerTable.remove(group); + if (remove != null) { + log.info("ungister consumer ok, no any connection, and remove consumer group, {}", group); + } + } + this.consumerIdsChangeListener.consumerIdsChanged(group, consumerGroupInfo.getAllChannel()); + } + } + + + public void scanNotActiveChannel() { + Iterator> it = this.consumerTable.entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + String group = next.getKey(); + ConsumerGroupInfo consumerGroupInfo = next.getValue(); + ConcurrentHashMap channelInfoTable = + consumerGroupInfo.getChannelInfoTable(); + + Iterator> itChannel = channelInfoTable.entrySet().iterator(); + while (itChannel.hasNext()) { + Entry nextChannel = itChannel.next(); + ClientChannelInfo clientChannelInfo = nextChannel.getValue(); + long diff = System.currentTimeMillis() - clientChannelInfo.getLastUpdateTimestamp(); + if (diff > ChannelExpiredTimeout) { + log.warn( + "SCAN: remove expired channel from ConsumerManager consumerTable. channel={}, consumerGroup={}", + RemotingHelper.parseChannelRemoteAddr(clientChannelInfo.getChannel()), group); + RemotingUtil.closeChannel(clientChannelInfo.getChannel()); + itChannel.remove(); + } + } + + if (channelInfoTable.isEmpty()) { + log.warn( + "SCAN: remove expired channel from ConsumerManager consumerTable, all clear, consumerGroup={}", + group); + it.remove(); + } + } + } + + + public HashSet queryTopicConsumeByWho(final String topic) { + HashSet groups = new HashSet(); + Iterator> it = this.consumerTable.entrySet().iterator(); + while (it.hasNext()) { + Entry entry = it.next(); + ConcurrentHashMap subscriptionTable = + entry.getValue().getSubscriptionTable(); + if (subscriptionTable.containsKey(topic)) { + groups.add(entry.getKey()); + } + } + + return groups; + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/client/DefaultConsumerIdsChangeListener.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/client/DefaultConsumerIdsChangeListener.java new file mode 100644 index 000000000..34c6b25fb --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/client/DefaultConsumerIdsChangeListener.java @@ -0,0 +1,47 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.broker.client; + +import com.alibaba.rocketmq.broker.BrokerController; +import io.netty.channel.Channel; + +import java.util.List; + + +/** + * ConsumerId列表变化，通知所有Consumer + * + * @author shijia.wxr + * @since 2013-7-26 + */ +public class DefaultConsumerIdsChangeListener implements ConsumerIdsChangeListener { + private final BrokerController brokerController; + + + public DefaultConsumerIdsChangeListener(BrokerController brokerController) { + this.brokerController = brokerController; + } + + + @Override + public void consumerIdsChanged(String group, List channels) { + if (channels != null && brokerController.getBrokerConfig().isNotifyConsumerIdsChangedEnable()) { + for (Channel chl : channels) { + this.brokerController.getBroker2Client().notifyConsumerIdsChanged(chl, group); + } + } + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/client/ProducerManager.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/client/ProducerManager.java new file mode 100644 index 000000000..1200e2793 --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/client/ProducerManager.java @@ -0,0 +1,201 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.broker.client; + +import io.netty.channel.Channel; + +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.Map.Entry; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.remoting.common.RemotingHelper; +import com.alibaba.rocketmq.remoting.common.RemotingUtil; + + +/** + * 管理Producer组及各个Producer连接 + * + * @author shijia.wxr + * @since 2013-7-26 + */ +public class ProducerManager { + private static final Logger log = LoggerFactory.getLogger(LoggerName.BrokerLoggerName); + private static final long LockTimeoutMillis = 3000; + private static final long ChannelExpiredTimeout = 1000 * 120; + private final Lock groupChannelLock = new ReentrantLock(); + private final HashMap> groupChannelTable = + new HashMap>(); + + + public ProducerManager() { + } + + + public HashMap> getGroupChannelTable() { + return groupChannelTable; + } + + + public void scanNotActiveChannel() { + try { + if (this.groupChannelLock.tryLock(LockTimeoutMillis, TimeUnit.MILLISECONDS)) { + try { + for (final Map.Entry> entry : this.groupChannelTable + .entrySet()) { + final String group = entry.getKey(); + final HashMap chlMap = entry.getValue(); + + Iterator> it = chlMap.entrySet().iterator(); + while (it.hasNext()) { + Entry item = it.next(); + // final Integer id = item.getKey(); + final ClientChannelInfo info = item.getValue(); + + long diff = System.currentTimeMillis() - info.getLastUpdateTimestamp(); + if (diff > ChannelExpiredTimeout) { + it.remove(); + log.warn( + "SCAN: remove expired channel[{}] from ProducerManager groupChannelTable, producer group name: {}", + RemotingHelper.parseChannelRemoteAddr(info.getChannel()), group); + RemotingUtil.closeChannel(info.getChannel()); + } + } + } + } + finally { + this.groupChannelLock.unlock(); + } + } + else { + log.warn("ProducerManager scanNotActiveChannel lock timeout"); + } + } + catch (InterruptedException e) { + log.error("", e); + } + } + + + public void doChannelCloseEvent(final String remoteAddr, final Channel channel) { + if (channel != null) { + try { + if (this.groupChannelLock.tryLock(LockTimeoutMillis, TimeUnit.MILLISECONDS)) { + try { + for (final Map.Entry> entry : this.groupChannelTable + .entrySet()) { + final String group = entry.getKey(); + final HashMap clientChannelInfoTable = + entry.getValue(); + final ClientChannelInfo clientChannelInfo = + clientChannelInfoTable.remove(channel); + if (clientChannelInfo != null) { + log.info( + "NETTY EVENT: remove channel[{}][{}] from ProducerManager groupChannelTable, producer group: {}", + clientChannelInfo.toString(), remoteAddr, group); + } + } + } + finally { + this.groupChannelLock.unlock(); + } + } + else { + log.warn("ProducerManager doChannelCloseEvent lock timeout"); + } + } + catch (InterruptedException e) { + log.error("", e); + } + } + } + + + public void registerProducer(final String group, final ClientChannelInfo clientChannelInfo) { + try { + ClientChannelInfo clientChannelInfoFound = null; + + if (this.groupChannelLock.tryLock(LockTimeoutMillis, TimeUnit.MILLISECONDS)) { + try { + HashMap channelTable = this.groupChannelTable.get(group); + if (null == channelTable) { + channelTable = new HashMap(); + this.groupChannelTable.put(group, channelTable); + } + + clientChannelInfoFound = channelTable.get(clientChannelInfo.getChannel()); + if (null == clientChannelInfoFound) { + channelTable.put(clientChannelInfo.getChannel(), clientChannelInfo); + log.info("new producer connected, group: {} channel: {}", group, + clientChannelInfo.toString()); + } + } + finally { + this.groupChannelLock.unlock(); + } + + if (clientChannelInfoFound != null) { + clientChannelInfoFound.setLastUpdateTimestamp(System.currentTimeMillis()); + } + } + else { + log.warn("ProducerManager registerProducer lock timeout"); + } + } + catch (InterruptedException e) { + log.error("", e); + } + } + + + public void unregisterProducer(final String group, final ClientChannelInfo clientChannelInfo) { + try { + if (this.groupChannelLock.tryLock(LockTimeoutMillis, TimeUnit.MILLISECONDS)) { + try { + HashMap channelTable = this.groupChannelTable.get(group); + if (null != channelTable && !channelTable.isEmpty()) { + ClientChannelInfo old = channelTable.remove(clientChannelInfo.getChannel()); + if (old != null) { + log.info("unregister a producer[{}] from groupChannelTable {}", group, + clientChannelInfo.toString()); + } + + if (channelTable.isEmpty()) { + this.groupChannelTable.remove(group); + log.info("unregister a producer group[{}] from groupChannelTable", group); + } + } + } + finally { + this.groupChannelLock.unlock(); + } + } + else { + log.warn("ProducerManager unregisterProducer lock timeout"); + } + } + catch (InterruptedException e) { + log.error("", e); + } + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/client/net/Broker2Client.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/client/net/Broker2Client.java new file mode 100644 index 000000000..c8739315a --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/client/net/Broker2Client.java @@ -0,0 +1,316 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.broker.client.net; + +import io.netty.channel.Channel; +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelFutureListener; +import io.netty.channel.FileRegion; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.rocketmq.broker.BrokerController; +import com.alibaba.rocketmq.broker.client.ClientChannelInfo; +import com.alibaba.rocketmq.broker.client.ConsumerGroupInfo; +import com.alibaba.rocketmq.broker.pagecache.OneMessageTransfer; +import com.alibaba.rocketmq.common.MQVersion; +import com.alibaba.rocketmq.common.TopicConfig; +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.common.protocol.RequestCode; +import com.alibaba.rocketmq.common.protocol.ResponseCode; +import com.alibaba.rocketmq.common.protocol.body.GetConsumerStatusBody; +import com.alibaba.rocketmq.common.protocol.body.ResetOffsetBody; +import com.alibaba.rocketmq.common.protocol.header.CheckTransactionStateRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.GetConsumerStatusRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.NotifyConsumerIdsChangedRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.ResetOffsetRequestHeader; +import com.alibaba.rocketmq.remoting.common.RemotingHelper; +import com.alibaba.rocketmq.remoting.exception.RemotingSendRequestException; +import com.alibaba.rocketmq.remoting.exception.RemotingTimeoutException; +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; +import com.alibaba.rocketmq.store.SelectMapedBufferResult; + + +/** + * Broker主动调用客户端接口 + * + * @author shijia.wxr + * @since 2013-7-26 + */ +public class Broker2Client { + private static final Logger log = LoggerFactory.getLogger(LoggerName.BrokerLoggerName); + private final BrokerController brokerController; + + + public Broker2Client(BrokerController brokerController) { + this.brokerController = brokerController; + } + + + /** + * Broker主动回查Producer事务状态，Oneway + */ + public void checkProducerTransactionState(// + final Channel channel,// + final CheckTransactionStateRequestHeader requestHeader,// + final SelectMapedBufferResult selectMapedBufferResult// + ) { + RemotingCommand request = + RemotingCommand.createRequestCommand(RequestCode.CHECK_TRANSACTION_STATE, requestHeader); + request.markOnewayRPC(); + + try { + FileRegion fileRegion = + new OneMessageTransfer(request.encodeHeader(selectMapedBufferResult.getSize()), + selectMapedBufferResult); + channel.writeAndFlush(fileRegion).addListener(new ChannelFutureListener() { + @Override + public void operationComplete(ChannelFuture future) throws Exception { + selectMapedBufferResult.release(); + if (!future.isSuccess()) { + log.error("invokeProducer failed,", future.cause()); + } + } + }); + } + catch (Throwable e) { + log.error("invokeProducer exception", e); + selectMapedBufferResult.release(); + } + } + + + public RemotingCommand callClient(// + final Channel channel,// + final RemotingCommand request// + ) throws RemotingSendRequestException, RemotingTimeoutException, InterruptedException { + return this.brokerController.getRemotingServer().invokeSync(channel, request, 10000); + } + + + /** + * Broker主动通知Consumer，Id列表发生变化，Oneway + */ + public void notifyConsumerIdsChanged(// + final Channel channel,// + final String consumerGroup// + ) { + if (null == consumerGroup) { + log.error("notifyConsumerIdsChanged consumerGroup is null"); + return; + } + + NotifyConsumerIdsChangedRequestHeader requestHeader = new NotifyConsumerIdsChangedRequestHeader(); + requestHeader.setConsumerGroup(consumerGroup); + RemotingCommand request = + RemotingCommand.createRequestCommand(RequestCode.NOTIFY_CONSUMER_IDS_CHANGED, requestHeader); + + try { + this.brokerController.getRemotingServer().invokeOneway(channel, request, 10); + } + catch (Exception e) { + log.error("notifyConsumerIdsChanged exception, " + consumerGroup, e); + } + } + + + /** + * Broker 主动通知 Consumer，offset 需要进行重置列表发生变化 + */ + public RemotingCommand resetOffset(String topic, String group, long timeStamp, boolean isForce) { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + + TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(topic); + if (null == topicConfig) { + log.error("[reset-offset] reset offset failed, no topic in this broker. topic={}", topic); + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("[reset-offset] reset offset failed, no topic in this broker. topic=" + topic); + return response; + } + + Map offsetTable = new HashMap(); + for (int i = 0; i < topicConfig.getWriteQueueNums(); i++) { + MessageQueue mq = new MessageQueue(); + mq.setBrokerName(this.brokerController.getBrokerConfig().getBrokerName()); + mq.setTopic(topic); + mq.setQueueId(i); + + long consumerOffset = + this.brokerController.getConsumerOffsetManager().queryOffset(group, topic, i); + if (-1 == consumerOffset) { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark(String.format("THe consumer group <%s> not exist", group)); + return response; + } + + long timeStampOffset = + this.brokerController.getMessageStore().getOffsetInQueueByTime(topic, i, timeStamp); + if (isForce || timeStampOffset < consumerOffset) { + offsetTable.put(mq, timeStampOffset); + } + else { + offsetTable.put(mq, consumerOffset); + } + } + + ResetOffsetRequestHeader requestHeader = new ResetOffsetRequestHeader(); + requestHeader.setTopic(topic); + requestHeader.setGroup(group); + requestHeader.setTimestamp(timeStamp); + RemotingCommand request = + RemotingCommand.createRequestCommand(RequestCode.RESET_CONSUMER_CLIENT_OFFSET, requestHeader); + ResetOffsetBody body = new ResetOffsetBody(); + body.setOffsetTable(offsetTable); + request.setBody(body.encode()); + + ConsumerGroupInfo consumerGroupInfo = + this.brokerController.getConsumerManager().getConsumerGroupInfo(group); + + // Consumer在线 + if (consumerGroupInfo != null && !consumerGroupInfo.getAllChannel().isEmpty()) { + ConcurrentHashMap channelInfoTable = + consumerGroupInfo.getChannelInfoTable(); + for (Channel channel : channelInfoTable.keySet()) { + int version = channelInfoTable.get(channel).getVersion(); + if (version >= MQVersion.Version.V3_0_7_SNAPSHOT.ordinal()) { + try { + this.brokerController.getRemotingServer().invokeOneway(channel, request, 5000); + log.info("[reset-offset] reset offset success. topic={}, group={}, clientId={}", + new Object[] { topic, group, channelInfoTable.get(channel).getClientId() }); + } + catch (Exception e) { + log.error("[reset-offset] reset offset exception. topic={}, group={}", + new Object[] { topic, group }, e); + } + } + else { + // 如果有一个客户端是不支持该功能的，则直接返回错误，需要应用方升级。 + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("the client does not support this feature. version=" + + MQVersion.getVersionDesc(version)); + log.warn("[reset-offset] the client does not support this feature. version={}", + RemotingHelper.parseChannelRemoteAddr(channel), MQVersion.getVersionDesc(version)); + return response; + } + } + } + // Consumer不在线 + else { + String errorInfo = + String.format( + "Consumer not online, so can not reset offset, Group: %s Topic: %s Timestamp: %d",// + requestHeader.getGroup(), // + requestHeader.getTopic(), // + requestHeader.getTimestamp()); + log.error(errorInfo); + response.setCode(ResponseCode.CONSUMER_NOT_ONLINE); + response.setRemark(errorInfo); + return response; + } + response.setCode(ResponseCode.SUCCESS); + ResetOffsetBody resBody = new ResetOffsetBody(); + resBody.setOffsetTable(offsetTable); + response.setBody(resBody.encode()); + return response; + } + + + /** + * Broker主动获取Consumer端的消息情况 + */ + public RemotingCommand getConsumeStatus(String topic, String group, String originClientId) { + final RemotingCommand result = RemotingCommand.createResponseCommand(null); + + GetConsumerStatusRequestHeader requestHeader = new GetConsumerStatusRequestHeader(); + requestHeader.setTopic(topic); + requestHeader.setGroup(group); + RemotingCommand request = + RemotingCommand.createRequestCommand(RequestCode.GET_CONSUMER_STATUS_FROM_CLIENT, + requestHeader); + + Map> consumerStatusTable = + new HashMap>(); + ConcurrentHashMap channelInfoTable = + this.brokerController.getConsumerManager().getConsumerGroupInfo(group).getChannelInfoTable(); + if (null == channelInfoTable || channelInfoTable.isEmpty()) { + result.setCode(ResponseCode.SYSTEM_ERROR); + result.setRemark(String.format("No Any Consumer online in the consumer group: [%s]", group)); + return result; + } + + for (Channel channel : channelInfoTable.keySet()) { + int version = channelInfoTable.get(channel).getVersion(); + String clientId = channelInfoTable.get(channel).getClientId(); + if (version < MQVersion.Version.V3_0_7_SNAPSHOT.ordinal()) { + // 如果有一个客户端是不支持该功能的，则直接返回错误，需要应用方升级。 + result.setCode(ResponseCode.SYSTEM_ERROR); + result.setRemark("the client does not support this feature. version=" + + MQVersion.getVersionDesc(version)); + log.warn("[get-consumer-status] the client does not support this feature. version={}", + RemotingHelper.parseChannelRemoteAddr(channel), MQVersion.getVersionDesc(version)); + return result; + } + else if (UtilAll.isBlank(originClientId) || originClientId.equals(clientId)) { + // 不指定 originClientId 则对所有的 client 进行处理；若指定 originClientId 则只对当前 + // originClientId 进行处理 + try { + RemotingCommand response = + this.brokerController.getRemotingServer().invokeSync(channel, request, 5000); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + if (response.getBody() != null) { + GetConsumerStatusBody body = + GetConsumerStatusBody.decode(response.getBody(), + GetConsumerStatusBody.class); + + consumerStatusTable.put(clientId, body.getMessageQueueTable()); + log.info( + "[get-consumer-status] get consumer status success. topic={}, group={}, channelRemoteAddr={}", + new Object[] { topic, group, clientId }); + } + } + default: + break; + } + } + catch (Exception e) { + log.error( + "[get-consumer-status] get consumer status exception. topic={}, group={}, offset={}", + new Object[] { topic, group }, e); + } + + // 若指定 originClientId 相应的 client 处理完成，则退出循环 + if (!UtilAll.isBlank(originClientId) && originClientId.equals(clientId)) { + break; + } + } + } + + result.setCode(ResponseCode.SUCCESS); + GetConsumerStatusBody resBody = new GetConsumerStatusBody(); + resBody.setConsumerTable(consumerStatusTable); + result.setBody(resBody.encode()); + return result; + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/client/rebalance/RebalanceLockManager.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/client/rebalance/RebalanceLockManager.java new file mode 100644 index 000000000..9fc55bb3f --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/client/rebalance/RebalanceLockManager.java @@ -0,0 +1,312 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.broker.client.rebalance; + +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.common.message.MessageQueue; + + +/** + * 顺序消息争抢队列锁 + * + * @author shijia.wxr + * @since 2013-6-26 + */ +public class RebalanceLockManager { + private static final Logger log = LoggerFactory.getLogger(LoggerName.RebalanceLockLoggerName); + private final static long RebalanceLockMaxLiveTime = Long.parseLong(System.getProperty( + "rocketmq.broker.rebalance.lockMaxLiveTime", "60000")); + private final Lock lock = new ReentrantLock(); + private final ConcurrentHashMap> mqLockTable = + new ConcurrentHashMap>(1024); + + class LockEntry { + private String clientId; + private volatile long lastUpdateTimestamp = System.currentTimeMillis(); + + + public String getClientId() { + return clientId; + } + + + public void setClientId(String clientId) { + this.clientId = clientId; + } + + + public long getLastUpdateTimestamp() { + return lastUpdateTimestamp; + } + + + public void setLastUpdateTimestamp(long lastUpdateTimestamp) { + this.lastUpdateTimestamp = lastUpdateTimestamp; + } + + + public boolean isExpired() { + boolean expired = + (System.currentTimeMillis() - this.lastUpdateTimestamp) > RebalanceLockMaxLiveTime; + + return expired; + } + + + public boolean isLocked(final String clientId) { + boolean eq = this.clientId.equals(clientId); + return eq && !this.isExpired(); + } + } + + + private boolean isLocked(final String group, final MessageQueue mq, final String clientId) { + ConcurrentHashMap groupValue = this.mqLockTable.get(group); + if (groupValue != null) { + LockEntry lockEntry = groupValue.get(mq); + if (lockEntry != null) { + boolean locked = lockEntry.isLocked(clientId); + if (locked) { + lockEntry.setLastUpdateTimestamp(System.currentTimeMillis()); + } + + return locked; + } + } + + return false; + } + + + /** + * 尝试锁队列 + * + * @return 是否lock成功 + */ + public boolean tryLock(final String group, final MessageQueue mq, final String clientId) { + // 没有被锁住 + if (!this.isLocked(group, mq, clientId)) { + try { + this.lock.lockInterruptibly(); + try { + ConcurrentHashMap groupValue = this.mqLockTable.get(group); + if (null == groupValue) { + groupValue = new ConcurrentHashMap(32); + this.mqLockTable.put(group, groupValue); + } + + LockEntry lockEntry = groupValue.get(mq); + if (null == lockEntry) { + lockEntry = new LockEntry(); + lockEntry.setClientId(clientId); + groupValue.put(mq, lockEntry); + log.info("tryLock, message queue not locked, I got it. Group: {} NewClientId: {} {}", // + group, // + clientId, // + mq); + } + + if (lockEntry.isLocked(clientId)) { + lockEntry.setLastUpdateTimestamp(System.currentTimeMillis()); + return true; + } + + String oldClientId = lockEntry.getClientId(); + + // 锁已经过期，抢占它 + if (lockEntry.isExpired()) { + lockEntry.setClientId(clientId); + lockEntry.setLastUpdateTimestamp(System.currentTimeMillis()); + log.warn( + "tryLock, message queue lock expired, I got it. Group: {} OldClientId: {} NewClientId: {} {}", // + group, // + oldClientId, // + clientId, // + mq); + return true; + } + + // 锁被别的Client占用 + log.warn( + "tryLock, message queue locked by other client. Group: {} OtherClientId: {} NewClientId: {} {}", // + group, // + oldClientId, // + clientId, // + mq); + return false; + } + finally { + this.lock.unlock(); + } + } + catch (InterruptedException e) { + log.error("putMessage exception", e); + } + } + // 已经锁住，尝试更新时间 + else { + // isLocked 中已经更新了时间，这里不需要再更新 + } + + return true; + } + + + /** + * 批量方式锁队列，返回锁定成功的队列集合 + * + * @return 是否lock成功 + */ + public Set tryLockBatch(final String group, final Set mqs, + final String clientId) { + Set lockedMqs = new HashSet(mqs.size()); + Set notLockedMqs = new HashSet(mqs.size()); + + // 先通过不加锁的方式尝试查看哪些锁定，哪些没锁定 + for (MessageQueue mq : mqs) { + if (this.isLocked(group, mq, clientId)) { + lockedMqs.add(mq); + } + else { + notLockedMqs.add(mq); + } + } + + if (!notLockedMqs.isEmpty()) { + try { + this.lock.lockInterruptibly(); + try { + ConcurrentHashMap groupValue = this.mqLockTable.get(group); + if (null == groupValue) { + groupValue = new ConcurrentHashMap(32); + this.mqLockTable.put(group, groupValue); + } + + // 遍历没有锁住的队列 + for (MessageQueue mq : notLockedMqs) { + LockEntry lockEntry = groupValue.get(mq); + if (null == lockEntry) { + lockEntry = new LockEntry(); + lockEntry.setClientId(clientId); + groupValue.put(mq, lockEntry); + log.info( + "tryLockBatch, message queue not locked, I got it. Group: {} NewClientId: {} {}", // + group, // + clientId, // + mq); + } + + // 已经锁定 + if (lockEntry.isLocked(clientId)) { + lockEntry.setLastUpdateTimestamp(System.currentTimeMillis()); + lockedMqs.add(mq); + continue; + } + + String oldClientId = lockEntry.getClientId(); + + // 锁已经过期，抢占它 + if (lockEntry.isExpired()) { + lockEntry.setClientId(clientId); + lockEntry.setLastUpdateTimestamp(System.currentTimeMillis()); + log.warn( + "tryLockBatch, message queue lock expired, I got it. Group: {} OldClientId: {} NewClientId: {} {}", // + group, // + oldClientId, // + clientId, // + mq); + lockedMqs.add(mq); + continue; + } + + // 锁被别的Client占用 + log.warn( + "tryLockBatch, message queue locked by other client. Group: {} OtherClientId: {} NewClientId: {} {}", // + group, // + oldClientId, // + clientId, // + mq); + } + } + finally { + this.lock.unlock(); + } + } + catch (InterruptedException e) { + log.error("putMessage exception", e); + } + } + + return lockedMqs; + } + + + public void unlockBatch(final String group, final Set mqs, final String clientId) { + try { + this.lock.lockInterruptibly(); + try { + ConcurrentHashMap groupValue = this.mqLockTable.get(group); + if (null != groupValue) { + for (MessageQueue mq : mqs) { + LockEntry lockEntry = groupValue.get(mq); + if (null != lockEntry) { + if (lockEntry.getClientId().equals(clientId)) { + groupValue.remove(mq); + log.info("unlockBatch, Group: {} {} {}",// + group, // + mq, // + clientId); + } + else { + log.warn("unlockBatch, but mq locked by other client: {}, Group: {} {} {}",// + lockEntry.getClientId(), // + group, // + mq, // + clientId); + } + } + else { + log.warn("unlockBatch, but mq not locked, Group: {} {} {}",// + group, // + mq, // + clientId); + } + } + } + else { + log.warn("unlockBatch, group not exist, Group: {} {}",// + group, // + clientId); + } + } + finally { + this.lock.unlock(); + } + } + catch (InterruptedException e) { + log.error("putMessage exception", e); + } + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/filtersrv/FilterServerManager.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/filtersrv/FilterServerManager.java new file mode 100644 index 000000000..913e77b5c --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/filtersrv/FilterServerManager.java @@ -0,0 +1,174 @@ +package com.alibaba.rocketmq.broker.filtersrv; + +import io.netty.channel.Channel; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map.Entry; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.rocketmq.broker.BrokerController; +import com.alibaba.rocketmq.broker.BrokerStartup; +import com.alibaba.rocketmq.common.ThreadFactoryImpl; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.remoting.common.RemotingUtil; + + +public class FilterServerManager { + private static final Logger log = LoggerFactory.getLogger(LoggerName.BrokerLoggerName); + // Filter Server最大空闲时间 + public static final long FilterServerMaxIdleTimeMills = 30000; + + private final ConcurrentHashMap filterServerTable = + new ConcurrentHashMap(16); + + private final BrokerController brokerController; + + private ScheduledExecutorService scheduledExecutorService = Executors + .newSingleThreadScheduledExecutor(new ThreadFactoryImpl("FilterServerManagerScheduledThread")); + + class FilterServerInfo { + private String filterServerAddr; + private long lastUpdateTimestamp; + + + public String getFilterServerAddr() { + return filterServerAddr; + } + + + public void setFilterServerAddr(String filterServerAddr) { + this.filterServerAddr = filterServerAddr; + } + + + public long getLastUpdateTimestamp() { + return lastUpdateTimestamp; + } + + + public void setLastUpdateTimestamp(long lastUpdateTimestamp) { + this.lastUpdateTimestamp = lastUpdateTimestamp; + } + } + + + public FilterServerManager(final BrokerController brokerController) { + this.brokerController = brokerController; + } + + + public void start() { + // 定时检查Filter Server个数，数量不符合，则创建 + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + try { + FilterServerManager.this.createFilterServer(); + } + catch (Exception e) { + log.error("", e); + } + } + }, 1000 * 5, 1000 * 30, TimeUnit.MILLISECONDS); + } + + + public void shutdown() { + this.scheduledExecutorService.shutdown(); + } + + + private String buildStartCommand() { + String config = ""; + if (BrokerStartup.configFile != null) { + config = String.format("-c %s", BrokerStartup.configFile); + } + + if (this.brokerController.getBrokerConfig().getNamesrvAddr() != null) { + config += String.format(" -n %s", this.brokerController.getBrokerConfig().getNamesrvAddr()); + } + + if (RemotingUtil.isWindowsPlatform()) { + return String.format("start /b %s\\bin\\mqfiltersrv.exe %s", // + this.brokerController.getBrokerConfig().getRocketmqHome(),// + config); + } + else { + return String.format("sh %s/bin/startfsrv.sh %s", // + this.brokerController.getBrokerConfig().getRocketmqHome(),// + config); + } + } + + + public void createFilterServer() { + int more = + this.brokerController.getBrokerConfig().getFilterServerNums() - this.filterServerTable.size(); + String cmd = this.buildStartCommand(); + for (int i = 0; i < more; i++) { + FilterServerUtil.callShell(cmd, log); + } + } + + + public void registerFilterServer(final Channel channel, final String filterServerAddr) { + FilterServerInfo filterServerInfo = this.filterServerTable.get(channel); + if (filterServerInfo != null) { + filterServerInfo.setLastUpdateTimestamp(System.currentTimeMillis()); + } + else { + filterServerInfo = new FilterServerInfo(); + filterServerInfo.setFilterServerAddr(filterServerAddr); + filterServerInfo.setLastUpdateTimestamp(System.currentTimeMillis()); + this.filterServerTable.put(channel, filterServerInfo); + log.info("Receive a New Filter Server<{}>", filterServerAddr); + } + } + + + /** + * Filter Server 10s向Broker注册一次，Broker如果发现30s没有注册，则删除它 + */ + public void scanNotActiveChannel() { + // 单位毫秒 + Iterator> it = this.filterServerTable.entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + long timestamp = next.getValue().getLastUpdateTimestamp(); + Channel channel = next.getKey(); + if ((System.currentTimeMillis() - timestamp) > FilterServerMaxIdleTimeMills) { + log.info("The Filter Server<{}> expired, remove it", next.getKey()); + it.remove(); + RemotingUtil.closeChannel(channel); + } + } + } + + + public void doChannelCloseEvent(final String remoteAddr, final Channel channel) { + FilterServerInfo old = this.filterServerTable.remove(channel); + if (old != null) { + log.warn("The Filter Server<{}> connection<{}> closed, remove it", old.getFilterServerAddr(), + remoteAddr); + } + } + + + public List buildNewFilterServerList() { + List addr = new ArrayList(); + Iterator> it = this.filterServerTable.entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + addr.add(next.getValue().getFilterServerAddr()); + } + return addr; + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/filtersrv/FilterServerUtil.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/filtersrv/FilterServerUtil.java new file mode 100644 index 000000000..42ac39d39 --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/filtersrv/FilterServerUtil.java @@ -0,0 +1,55 @@ +package com.alibaba.rocketmq.broker.filtersrv; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; + +import org.slf4j.Logger; + + +public class FilterServerUtil { + + private static String[] splitShellString(final String shellString) { + String[] split = shellString.split(" "); + return split; + } + + + public static void callShell(final String shellString, final Logger log) { + Process process = null; + try { + String[] cmdArray = splitShellString(shellString); + process = Runtime.getRuntime().exec(cmdArray); + final InputStream is1 = process.getInputStream(); + new Thread(new Runnable() { + public void run() { + BufferedReader br = new BufferedReader(new InputStreamReader(is1)); + try { + while (br.readLine() != null) + ; + } + catch (IOException e) { + log.error("callShell: readLine IOException, " + shellString, e); + } + } + }).start(); // 启动单独的线程来清空process.getInputStream()的缓冲区 + + InputStream is2 = process.getErrorStream(); + BufferedReader br2 = new BufferedReader(new InputStreamReader(is2)); + StringBuilder buf = new StringBuilder(); // 保存输出结果流 + String line = null; + while ((line = br2.readLine()) != null) + buf.append(line); // 循环等待ffmpeg进程结束 + + log.info("callShell: <{}> OK", shellString); + } + catch (Throwable e) { + log.error("callShell: readLine IOException, " + shellString, e); + } + finally { + if (null != process) + process.destroy(); + } + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/longpolling/ManyPullRequest.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/longpolling/ManyPullRequest.java new file mode 100644 index 000000000..a001b9615 --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/longpolling/ManyPullRequest.java @@ -0,0 +1,51 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.broker.longpolling; + +import java.util.ArrayList; +import java.util.List; + + +/** + * 长轮询请求 + * + * @author shijia.wxr + * @since 2013-7-26 + */ +public class ManyPullRequest { + private final ArrayList pullRequestList = new ArrayList(); + + + public synchronized void addPullRequest(final PullRequest pullRequest) { + this.pullRequestList.add(pullRequest); + } + + + public synchronized void addPullRequest(final List many) { + this.pullRequestList.addAll(many); + } + + + public synchronized List cloneListAndClear() { + if (!this.pullRequestList.isEmpty()) { + List result = (ArrayList) this.pullRequestList.clone(); + this.pullRequestList.clear(); + return result; + } + + return null; + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/longpolling/PullRequest.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/longpolling/PullRequest.java new file mode 100644 index 000000000..3d578d29d --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/longpolling/PullRequest.java @@ -0,0 +1,70 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.broker.longpolling; + +import io.netty.channel.Channel; + +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; + + +/** + * 一个拉消息请求 + * + * @author shijia.wxr + * @since 2013-7-26 + */ +public class PullRequest { + private final RemotingCommand requestCommand; + private final Channel clientChannel; + private final long timeoutMillis; + private final long suspendTimestamp; + private final long pullFromThisOffset; + + + public PullRequest(RemotingCommand requestCommand, Channel clientChannel, long timeoutMillis, + long suspendTimestamp, long pullFromThisOffset) { + this.requestCommand = requestCommand; + this.clientChannel = clientChannel; + this.timeoutMillis = timeoutMillis; + this.suspendTimestamp = suspendTimestamp; + this.pullFromThisOffset = pullFromThisOffset; + } + + + public RemotingCommand getRequestCommand() { + return requestCommand; + } + + + public Channel getClientChannel() { + return clientChannel; + } + + + public long getTimeoutMillis() { + return timeoutMillis; + } + + + public long getSuspendTimestamp() { + return suspendTimestamp; + } + + + public long getPullFromThisOffset() { + return pullFromThisOffset; + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/longpolling/PullRequestHoldService.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/longpolling/PullRequestHoldService.java new file mode 100644 index 000000000..7953473ca --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/longpolling/PullRequestHoldService.java @@ -0,0 +1,172 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.broker.longpolling; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.rocketmq.broker.BrokerController; +import com.alibaba.rocketmq.common.ServiceThread; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * 拉消息请求管理，如果拉不到消息，则在这里Hold住，等待消息到来 + * + * @author shijia.wxr + * @since 2013-7-26 + */ +public class PullRequestHoldService extends ServiceThread { + private static final Logger log = LoggerFactory.getLogger(LoggerName.BrokerLoggerName); + private static final String TOPIC_QUEUEID_SEPARATOR = "@"; + + private ConcurrentHashMap pullRequestTable = + new ConcurrentHashMap(1024); + + private final BrokerController brokerController; + + + public PullRequestHoldService(final BrokerController brokerController) { + this.brokerController = brokerController; + } + + + private String buildKey(final String topic, final int queueId) { + StringBuilder sb = new StringBuilder(); + sb.append(topic); + sb.append(TOPIC_QUEUEID_SEPARATOR); + sb.append(queueId); + return sb.toString(); + } + + + public void suspendPullRequest(final String topic, final int queueId, final PullRequest pullRequest) { + String key = this.buildKey(topic, queueId); + ManyPullRequest mpr = this.pullRequestTable.get(key); + if (null == mpr) { + mpr = new ManyPullRequest(); + ManyPullRequest prev = this.pullRequestTable.putIfAbsent(key, mpr); + if (prev != null) { + mpr = prev; + } + } + + mpr.addPullRequest(pullRequest); + } + + + private void checkHoldRequest() { + for (String key : this.pullRequestTable.keySet()) { + String[] kArray = key.split(TOPIC_QUEUEID_SEPARATOR); + if (kArray != null && 2 == kArray.length) { + String topic = kArray[0]; + int queueId = Integer.parseInt(kArray[1]); + final long offset = + this.brokerController.getMessageStore().getMaxOffsetInQuque(topic, queueId); + this.notifyMessageArriving(topic, queueId, offset); + } + } + } + + + public void notifyMessageArriving(final String topic, final int queueId, final long maxOffset) { + String key = this.buildKey(topic, queueId); + ManyPullRequest mpr = this.pullRequestTable.get(key); + if (mpr != null) { + List requestList = mpr.cloneListAndClear(); + if (requestList != null) { + List replayList = new ArrayList(); + + for (PullRequest request : requestList) { + // 查看是否offset OK + if (maxOffset > request.getPullFromThisOffset()) { + try { + this.brokerController.getPullMessageProcessor().excuteRequestWhenWakeup( + request.getClientChannel(), request.getRequestCommand()); + } + catch (RemotingCommandException e) { + log.error("", e); + } + continue; + } + // 尝试取最新Offset + else { + final long newestOffset = + this.brokerController.getMessageStore().getMaxOffsetInQuque(topic, queueId); + if (newestOffset > request.getPullFromThisOffset()) { + try { + this.brokerController.getPullMessageProcessor().excuteRequestWhenWakeup( + request.getClientChannel(), request.getRequestCommand()); + } + catch (RemotingCommandException e) { + log.error("", e); + } + continue; + } + } + + // 查看是否超时 + if (System.currentTimeMillis() >= (request.getSuspendTimestamp() + request + .getTimeoutMillis())) { + try { + this.brokerController.getPullMessageProcessor().excuteRequestWhenWakeup( + request.getClientChannel(), request.getRequestCommand()); + } + catch (RemotingCommandException e) { + log.error("", e); + } + continue; + } + + // 当前不满足要求，重新放回Hold列表中 + replayList.add(request); + } + + if (!replayList.isEmpty()) { + mpr.addPullRequest(replayList); + } + } + } + } + + + @Override + public void run() { + log.info(this.getServiceName() + " service started"); + while (!this.isStoped()) { + try { + this.waitForRunning(1000); + this.checkHoldRequest(); + } + catch (Exception e) { + log.warn(this.getServiceName() + " service has exception. ", e); + } + } + + log.info(this.getServiceName() + " service end"); + } + + + @Override + public String getServiceName() { + return PullRequestHoldService.class.getSimpleName(); + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/mqtrace/ConsumeMessageContext.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/mqtrace/ConsumeMessageContext.java new file mode 100644 index 000000000..7e494505f --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/mqtrace/ConsumeMessageContext.java @@ -0,0 +1,132 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.broker.mqtrace; + +import java.util.Map; + + +public class ConsumeMessageContext { + private String consumerGroup; + private String topic; + private Integer queueId; + private String clientHost; + private String storeHost; + private Map messageIds; + private int bodyLength; + private boolean success; + private String status; + private Object mqTraceContext; + + + public String getConsumerGroup() { + return consumerGroup; + } + + + public void setConsumerGroup(String consumerGroup) { + this.consumerGroup = consumerGroup; + } + + + public String getTopic() { + return topic; + } + + + public void setTopic(String topic) { + this.topic = topic; + } + + + public Integer getQueueId() { + return queueId; + } + + + public void setQueueId(Integer queueId) { + this.queueId = queueId; + } + + + public String getClientHost() { + return clientHost; + } + + + public void setClientHost(String clientHost) { + this.clientHost = clientHost; + } + + + public String getStoreHost() { + return storeHost; + } + + + public void setStoreHost(String storeHost) { + this.storeHost = storeHost; + } + + + public Map getMessageIds() { + return messageIds; + } + + + public void setMessageIds(Map messageIds) { + this.messageIds = messageIds; + } + + + public boolean isSuccess() { + return success; + } + + + public void setSuccess(boolean success) { + this.success = success; + } + + + public String getStatus() { + return status; + } + + + public void setStatus(String status) { + this.status = status; + } + + + public Object getMqTraceContext() { + return mqTraceContext; + } + + + public void setMqTraceContext(Object mqTraceContext) { + this.mqTraceContext = mqTraceContext; + } + + + public int getBodyLength() { + return bodyLength; + } + + + public void setBodyLength(int bodyLength) { + this.bodyLength = bodyLength; + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/mqtrace/ConsumeMessageHook.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/mqtrace/ConsumeMessageHook.java new file mode 100644 index 000000000..76e35e483 --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/mqtrace/ConsumeMessageHook.java @@ -0,0 +1,26 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.broker.mqtrace; + +public interface ConsumeMessageHook { + public String hookName(); + + + public void consumeMessageBefore(final ConsumeMessageContext context); + + + public void consumeMessageAfter(final ConsumeMessageContext context); +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/mqtrace/SendMessageContext.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/mqtrace/SendMessageContext.java new file mode 100644 index 000000000..391433190 --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/mqtrace/SendMessageContext.java @@ -0,0 +1,176 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.broker.mqtrace; + +import java.util.Properties; + + +public class SendMessageContext { + private String producerGroup; + private String topic; + private String msgId; + private String originMsgId; + private Integer queueId; + private Long queueOffset; + private String brokerAddr; + private String bornHost; + private int bodyLength; + private int code; + private String errorMsg; + private String msgProps; + private Object mqTraceContext; + private Properties extProps; + + + public String getProducerGroup() { + return producerGroup; + } + + + public void setProducerGroup(String producerGroup) { + this.producerGroup = producerGroup; + } + + + public String getTopic() { + return topic; + } + + + public void setTopic(String topic) { + this.topic = topic; + } + + + public String getMsgId() { + return msgId; + } + + + public void setMsgId(String msgId) { + this.msgId = msgId; + } + + + public String getOriginMsgId() { + return originMsgId; + } + + + public void setOriginMsgId(String originMsgId) { + this.originMsgId = originMsgId; + } + + + public Integer getQueueId() { + return queueId; + } + + + public void setQueueId(Integer queueId) { + this.queueId = queueId; + } + + + public Long getQueueOffset() { + return queueOffset; + } + + + public void setQueueOffset(Long queueOffset) { + this.queueOffset = queueOffset; + } + + + public String getBrokerAddr() { + return brokerAddr; + } + + + public void setBrokerAddr(String brokerAddr) { + this.brokerAddr = brokerAddr; + } + + + public String getBornHost() { + return bornHost; + } + + + public void setBornHost(String bornHost) { + this.bornHost = bornHost; + } + + + public int getBodyLength() { + return bodyLength; + } + + + public void setBodyLength(int bodyLength) { + this.bodyLength = bodyLength; + } + + + public int getCode() { + return code; + } + + + public void setCode(int code) { + this.code = code; + } + + + public String getErrorMsg() { + return errorMsg; + } + + + public void setErrorMsg(String errorMsg) { + this.errorMsg = errorMsg; + } + + + public String getMsgProps() { + return msgProps; + } + + + public void setMsgProps(String msgProps) { + this.msgProps = msgProps; + } + + + public Object getMqTraceContext() { + return mqTraceContext; + } + + + public void setMqTraceContext(Object mqTraceContext) { + this.mqTraceContext = mqTraceContext; + } + + + public Properties getExtProps() { + return extProps; + } + + + public void setExtProps(Properties extProps) { + this.extProps = extProps; + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/mqtrace/SendMessageHook.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/mqtrace/SendMessageHook.java new file mode 100644 index 000000000..ce3a0fe28 --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/mqtrace/SendMessageHook.java @@ -0,0 +1,26 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.broker.mqtrace; + +public interface SendMessageHook { + public String hookName(); + + + public void sendMessageBefore(final SendMessageContext context); + + + public void sendMessageAfter(final SendMessageContext context); +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/offset/ConsumerOffsetManager.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/offset/ConsumerOffsetManager.java new file mode 100644 index 000000000..a85368a03 --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/offset/ConsumerOffsetManager.java @@ -0,0 +1,265 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.broker.offset; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.rocketmq.broker.BrokerController; +import com.alibaba.rocketmq.broker.BrokerPathConfigHelper; +import com.alibaba.rocketmq.common.ConfigManager; +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; + + +/** + * Consumer消费进度管理 + * + * @author shijia.wxr + * @since 2013-8-11 + */ +public class ConsumerOffsetManager extends ConfigManager { + private static final Logger log = LoggerFactory.getLogger(LoggerName.BrokerLoggerName); + private static final String TOPIC_GROUP_SEPARATOR = "@"; + + private ConcurrentHashMap> offsetTable = + new ConcurrentHashMap>(512); + + private transient BrokerController brokerController; + + + public ConsumerOffsetManager() { + } + + + public ConsumerOffsetManager(BrokerController brokerController) { + this.brokerController = brokerController; + } + + + /** + * 扫描数据被删除了的topic，offset记录也对应删除 + */ + public void scanUnsubscribedTopic() { + Iterator>> it = this.offsetTable.entrySet().iterator(); + while (it.hasNext()) { + Entry> next = it.next(); + String topicAtGroup = next.getKey(); + String[] arrays = topicAtGroup.split(TOPIC_GROUP_SEPARATOR); + if (arrays != null && arrays.length == 2) { + String topic = arrays[0]; + String group = arrays[1]; + // 当前订阅关系里面没有group-topic订阅关系（消费端当前是停机的状态）并且offset落后很多,则删除消费进度 + if (null == brokerController.getConsumerManager().findSubscriptionData(group, topic) + && this.offsetBehindMuchThanData(topic, next.getValue())) { + it.remove(); + log.warn("remove topic offset, {}", topicAtGroup); + } + } + } + } + + + private boolean offsetBehindMuchThanData(final String topic, ConcurrentHashMap table) { + Iterator> it = table.entrySet().iterator(); + boolean result = !table.isEmpty(); + + while (it.hasNext() && result) { + Entry next = it.next(); + long minOffsetInStore = + this.brokerController.getMessageStore().getMinOffsetInQuque(topic, next.getKey()); + long offsetInPersist = next.getValue(); + if (offsetInPersist > minOffsetInStore) { + result = false; + } + } + + return result; + } + + + public Set whichTopicByConsumer(final String group) { + Set topics = new HashSet(); + + Iterator>> it = this.offsetTable.entrySet().iterator(); + while (it.hasNext()) { + Entry> next = it.next(); + String topicAtGroup = next.getKey(); + String[] arrays = topicAtGroup.split(TOPIC_GROUP_SEPARATOR); + if (arrays != null && arrays.length == 2) { + if (group.equals(arrays[1])) { + topics.add(arrays[0]); + } + } + } + + return topics; + } + + + public Set whichGroupByTopic(final String topic) { + Set groups = new HashSet(); + + Iterator>> it = this.offsetTable.entrySet().iterator(); + while (it.hasNext()) { + Entry> next = it.next(); + String topicAtGroup = next.getKey(); + String[] arrays = topicAtGroup.split(TOPIC_GROUP_SEPARATOR); + if (arrays != null && arrays.length == 2) { + if (topic.equals(arrays[0])) { + groups.add(arrays[1]); + } + } + } + + return groups; + } + + + public void commitOffset(final String group, final String topic, final int queueId, final long offset) { + // topic@group + String key = topic + TOPIC_GROUP_SEPARATOR + group; + this.commitOffset(key, queueId, offset); + } + + + public long queryOffset(final String group, final String topic, final int queueId) { + // topic@group + String key = topic + TOPIC_GROUP_SEPARATOR + group; + ConcurrentHashMap map = this.offsetTable.get(key); + if (null != map) { + Long offset = map.get(queueId); + if (offset != null) + return offset; + } + + return -1; + } + + + private void commitOffset(final String key, final int queueId, final long offset) { + ConcurrentHashMap map = this.offsetTable.get(key); + if (null == map) { + map = new ConcurrentHashMap(32); + map.put(queueId, offset); + this.offsetTable.put(key, map); + } + else { + map.put(queueId, offset); + } + } + + + public String encode() { + return this.encode(false); + } + + + public String encode(final boolean prettyFormat) { + return RemotingSerializable.toJson(this, prettyFormat); + } + + + @Override + public void decode(String jsonString) { + if (jsonString != null) { + ConsumerOffsetManager obj = + RemotingSerializable.fromJson(jsonString, ConsumerOffsetManager.class); + if (obj != null) { + this.offsetTable = obj.offsetTable; + } + } + } + + + @Override + public String configFilePath() { + return BrokerPathConfigHelper.getConsumerOffsetPath(this.brokerController.getMessageStoreConfig() + .getStorePathRootDir()); + } + + + public ConcurrentHashMap> getOffsetTable() { + return offsetTable; + } + + + public void setOffsetTable(ConcurrentHashMap> offsetTable) { + this.offsetTable = offsetTable; + } + + + public Map queryMinOffsetInAllGroup(final String topic, final String filterGroups) { + + Map queueMinOffset = new HashMap(); + Set topicGroups = this.offsetTable.keySet(); + if (!UtilAll.isBlank(filterGroups)) { + for (String group : filterGroups.split(",")) { + Iterator it = topicGroups.iterator(); + while (it.hasNext()) { + if (group.equals(it.next().split(TOPIC_GROUP_SEPARATOR)[1])) { + it.remove(); + } + } + } + } + for (String topicGroup : topicGroups) { + String[] topicGroupArr = topicGroup.split(TOPIC_GROUP_SEPARATOR); + if (topic.equals(topicGroupArr[0])) { + for (Entry entry : this.offsetTable.get(topicGroup).entrySet()) { + long minOffset = + this.brokerController.getMessageStore() + .getMinOffsetInQuque(topic, entry.getKey()); + if (entry.getValue() >= minOffset) { + Long offset = queueMinOffset.get(entry.getKey()); + if (offset == null) { + queueMinOffset.put(entry.getKey(), Math.min(Long.MAX_VALUE, entry.getValue())); + } + else { + queueMinOffset.put(entry.getKey(), Math.min(entry.getValue(), offset)); + } + } + } + } + } + return queueMinOffset; + } + + + public Map queryOffset(final String group, final String topic) { + // topic@group + String key = topic + TOPIC_GROUP_SEPARATOR + group; + return this.offsetTable.get(key); + } + + + public void cloneOffset(final String srcGroup, final String destGroup, final String topic) { + ConcurrentHashMap offsets = + this.offsetTable.get(topic + TOPIC_GROUP_SEPARATOR + srcGroup); + if (offsets != null) { + this.offsetTable.put(topic + TOPIC_GROUP_SEPARATOR + destGroup, offsets); + } + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/out/BrokerOuterAPI.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/out/BrokerOuterAPI.java new file mode 100644 index 000000000..00ed62a73 --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/out/BrokerOuterAPI.java @@ -0,0 +1,348 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.broker.out; + +import java.util.ArrayList; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.rocketmq.client.exception.MQBrokerException; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.common.namesrv.RegisterBrokerResult; +import com.alibaba.rocketmq.common.namesrv.TopAddressing; +import com.alibaba.rocketmq.common.protocol.RequestCode; +import com.alibaba.rocketmq.common.protocol.ResponseCode; +import com.alibaba.rocketmq.common.protocol.body.ConsumerOffsetSerializeWrapper; +import com.alibaba.rocketmq.common.protocol.body.KVTable; +import com.alibaba.rocketmq.common.protocol.body.RegisterBrokerBody; +import com.alibaba.rocketmq.common.protocol.body.SubscriptionGroupWrapper; +import com.alibaba.rocketmq.common.protocol.body.TopicConfigSerializeWrapper; +import com.alibaba.rocketmq.common.protocol.header.namesrv.RegisterBrokerRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.namesrv.RegisterBrokerResponseHeader; +import com.alibaba.rocketmq.common.protocol.header.namesrv.UnRegisterBrokerRequestHeader; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.remoting.RemotingClient; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; +import com.alibaba.rocketmq.remoting.exception.RemotingConnectException; +import com.alibaba.rocketmq.remoting.exception.RemotingSendRequestException; +import com.alibaba.rocketmq.remoting.exception.RemotingTimeoutException; +import com.alibaba.rocketmq.remoting.netty.NettyClientConfig; +import com.alibaba.rocketmq.remoting.netty.NettyRemotingClient; +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; + + +/** + * Broker对外调用的API封装 + * + * @author shijia.wxr + * @author manhong.yqd + * @since 2013-7-3 + */ +public class BrokerOuterAPI { + private static final Logger log = LoggerFactory.getLogger(LoggerName.BrokerLoggerName); + private final RemotingClient remotingClient; + private final TopAddressing topAddressing = new TopAddressing(MixAll.WS_ADDR); + private String nameSrvAddr = null; + + + public BrokerOuterAPI(final NettyClientConfig nettyClientConfig, RPCHook rpcHook) { + this.remotingClient = new NettyRemotingClient(nettyClientConfig); + this.remotingClient.registerRPCHook(rpcHook); + } + + + public BrokerOuterAPI(final NettyClientConfig nettyClientConfig) { + this(nettyClientConfig, null); + } + + + public void start() { + this.remotingClient.start(); + } + + + public void shutdown() { + this.remotingClient.shutdown(); + } + + + public String fetchNameServerAddr() { + try { + String addrs = this.topAddressing.fetchNSAddr(); + if (addrs != null) { + if (!addrs.equals(this.nameSrvAddr)) { + log.info("name server address changed, old: " + this.nameSrvAddr + " new: " + addrs); + this.updateNameServerAddressList(addrs); + this.nameSrvAddr = addrs; + return nameSrvAddr; + } + } + } + catch (Exception e) { + log.error("fetchNameServerAddr Exception", e); + } + return nameSrvAddr; + } + + + public void updateNameServerAddressList(final String addrs) { + List lst = new ArrayList(); + String[] addrArray = addrs.split(";"); + if (addrArray != null) { + for (String addr : addrArray) { + lst.add(addr); + } + + this.remotingClient.updateNameServerAddressList(lst); + } + } + + + private RegisterBrokerResult registerBroker(// + final String namesrvAddr,// + final String clusterName,// 1 + final String brokerAddr,// 2 + final String brokerName,// 3 + final long brokerId,// 4 + final String haServerAddr,// 5 + final TopicConfigSerializeWrapper topicConfigWrapper, // 6 + final List filterServerList // 7 + ) throws RemotingCommandException, MQBrokerException, RemotingConnectException, + RemotingSendRequestException, RemotingTimeoutException, InterruptedException { + RegisterBrokerRequestHeader requestHeader = new RegisterBrokerRequestHeader(); + requestHeader.setBrokerAddr(brokerAddr); + requestHeader.setBrokerId(brokerId); + requestHeader.setBrokerName(brokerName); + requestHeader.setClusterName(clusterName); + requestHeader.setHaServerAddr(haServerAddr); + RemotingCommand request = + RemotingCommand.createRequestCommand(RequestCode.REGISTER_BROKER, requestHeader); + + RegisterBrokerBody requestBody = new RegisterBrokerBody(); + requestBody.setTopicConfigSerializeWrapper(topicConfigWrapper); + requestBody.setFilterServerList(filterServerList); + request.setBody(requestBody.encode()); + + RemotingCommand response = this.remotingClient.invokeSync(namesrvAddr, request, 3000); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + RegisterBrokerResponseHeader responseHeader = + (RegisterBrokerResponseHeader) response + .decodeCommandCustomHeader(RegisterBrokerResponseHeader.class); + RegisterBrokerResult result = new RegisterBrokerResult(); + result.setMasterAddr(responseHeader.getMasterAddr()); + result.setHaServerAddr(responseHeader.getHaServerAddr()); + result.setHaServerAddr(responseHeader.getHaServerAddr()); + if (response.getBody() != null) { + result.setKvTable(KVTable.decode(response.getBody(), KVTable.class)); + } + return result; + } + default: + break; + } + + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + + + public RegisterBrokerResult registerBrokerAll(// + final String clusterName,// 1 + final String brokerAddr,// 2 + final String brokerName,// 3 + final long brokerId,// 4 + final String haServerAddr,// 5 + final TopicConfigSerializeWrapper topicConfigWrapper,// 6 + final List filterServerList // 7 + ) { + RegisterBrokerResult registerBrokerResult = null; + + List nameServerAddressList = this.remotingClient.getNameServerAddressList(); + if (nameServerAddressList != null) { + for (String namesrvAddr : nameServerAddressList) { + try { + RegisterBrokerResult result = + this.registerBroker(namesrvAddr, clusterName, brokerAddr, brokerName, brokerId, + haServerAddr, topicConfigWrapper, filterServerList); + if (result != null) { + registerBrokerResult = result; + } + + log.info("register broker to name server {} OK", namesrvAddr); + } + catch (Exception e) { + log.warn("registerBroker Exception, " + namesrvAddr, e); + } + } + } + + return registerBrokerResult; + } + + + public void unregisterBroker(// + final String namesrvAddr,// + final String clusterName,// 1 + final String brokerAddr,// 2 + final String brokerName,// 3 + final long brokerId// 4 + ) throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, + InterruptedException, MQBrokerException { + UnRegisterBrokerRequestHeader requestHeader = new UnRegisterBrokerRequestHeader(); + requestHeader.setBrokerAddr(brokerAddr); + requestHeader.setBrokerId(brokerId); + requestHeader.setBrokerName(brokerName); + requestHeader.setClusterName(clusterName); + RemotingCommand request = + RemotingCommand.createRequestCommand(RequestCode.UNREGISTER_BROKER, requestHeader); + + RemotingCommand response = this.remotingClient.invokeSync(namesrvAddr, request, 3000); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + return; + } + default: + break; + } + + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + + + public void unregisterBrokerAll(// + final String clusterName,// 1 + final String brokerAddr,// 2 + final String brokerName,// 3 + final long brokerId// 4 + ) { + List nameServerAddressList = this.remotingClient.getNameServerAddressList(); + if (nameServerAddressList != null) { + for (String namesrvAddr : nameServerAddressList) { + try { + this.unregisterBroker(namesrvAddr, clusterName, brokerAddr, brokerName, brokerId); + log.info("unregisterBroker OK, NamesrvAddr: {}", namesrvAddr); + } + catch (Exception e) { + log.warn("unregisterBroker Exception, " + namesrvAddr, e); + } + } + } + } + + + public TopicConfigSerializeWrapper getAllTopicConfig(final String addr) throws RemotingConnectException, + RemotingSendRequestException, RemotingTimeoutException, InterruptedException, MQBrokerException { + RemotingCommand request = + RemotingCommand.createRequestCommand(RequestCode.GET_ALL_TOPIC_CONFIG, null); + + RemotingCommand response = this.remotingClient.invokeSync(addr, request, 3000); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + return TopicConfigSerializeWrapper.decode(response.getBody(), TopicConfigSerializeWrapper.class); + } + default: + break; + } + + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + + + /** + * 获取所有Consumer Offset + * + * @param addr + * @return + */ + public ConsumerOffsetSerializeWrapper getAllConsumerOffset(final String addr) + throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, + RemotingConnectException, MQBrokerException { + RemotingCommand request = + RemotingCommand.createRequestCommand(RequestCode.GET_ALL_CONSUMER_OFFSET, null); + RemotingCommand response = this.remotingClient.invokeSync(addr, request, 3000); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + return ConsumerOffsetSerializeWrapper.decode(response.getBody(), + ConsumerOffsetSerializeWrapper.class); + } + default: + break; + } + + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + + + /** + * 获取所有定时进度 + * + * @param addr + * @return + */ + public String getAllDelayOffset(final String addr) throws InterruptedException, RemotingTimeoutException, + RemotingSendRequestException, RemotingConnectException, MQBrokerException { + RemotingCommand request = + RemotingCommand.createRequestCommand(RequestCode.GET_ALL_DELAY_OFFSET, null); + RemotingCommand response = this.remotingClient.invokeSync(addr, request, 3000); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + return new String(response.getBody()); + } + default: + break; + } + + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + + + /** + * 获取订阅组配置 + * + * @param addr + * @return + */ + public SubscriptionGroupWrapper getAllSubscriptionGroupConfig(final String addr) + throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, + RemotingConnectException, MQBrokerException { + RemotingCommand request = + RemotingCommand.createRequestCommand(RequestCode.GET_ALL_SUBSCRIPTIONGROUP_CONFIG, null); + RemotingCommand response = this.remotingClient.invokeSync(addr, request, 3000); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + return SubscriptionGroupWrapper.decode(response.getBody(), SubscriptionGroupWrapper.class); + } + default: + break; + } + + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + + + public void registerRPCHook(RPCHook rpcHook) { + remotingClient.registerRPCHook(rpcHook); + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/pagecache/ManyMessageTransfer.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/pagecache/ManyMessageTransfer.java new file mode 100644 index 000000000..aa7888c9c --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/pagecache/ManyMessageTransfer.java @@ -0,0 +1,97 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.broker.pagecache; + +import io.netty.channel.FileRegion; +import io.netty.util.AbstractReferenceCounted; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.WritableByteChannel; +import java.util.List; + +import com.alibaba.rocketmq.store.GetMessageResult; + + +/** + * @author shijia.wxr + * @since 2013-7-26 + */ +public class ManyMessageTransfer extends AbstractReferenceCounted implements FileRegion { + private final ByteBuffer byteBufferHeader; + private final GetMessageResult getMessageResult; + private long transfered; // the bytes which was transfered already + + + public ManyMessageTransfer(ByteBuffer byteBufferHeader, GetMessageResult getMessageResult) { + this.byteBufferHeader = byteBufferHeader; + this.getMessageResult = getMessageResult; + } + + + @Override + public long position() { + int pos = byteBufferHeader.position(); + List messageBufferList = this.getMessageResult.getMessageBufferList(); + for (ByteBuffer bb : messageBufferList) { + pos += bb.position(); + } + return pos; + } + + + @Override + public long count() { + return byteBufferHeader.limit() + this.getMessageResult.getBufferTotalSize(); + } + + + @Override + public long transferTo(WritableByteChannel target, long position) throws IOException { + if (this.byteBufferHeader.hasRemaining()) { + transfered += target.write(this.byteBufferHeader); + return transfered; + } + else { + List messageBufferList = this.getMessageResult.getMessageBufferList(); + for (ByteBuffer bb : messageBufferList) { + if (bb.hasRemaining()) { + transfered += target.write(bb); + return transfered; + } + } + } + + return 0; + } + + + public void close() { + this.deallocate(); + } + + + @Override + protected void deallocate() { + this.getMessageResult.release(); + } + + + @Override + public long transfered() { + return transfered; + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/pagecache/OneMessageTransfer.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/pagecache/OneMessageTransfer.java new file mode 100644 index 000000000..4f0f46eed --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/pagecache/OneMessageTransfer.java @@ -0,0 +1,86 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.broker.pagecache; + +import io.netty.channel.FileRegion; +import io.netty.util.AbstractReferenceCounted; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.WritableByteChannel; + +import com.alibaba.rocketmq.store.SelectMapedBufferResult; + + +/** + * @author shijia.wxr + * @since 2013-7-26 + */ +public class OneMessageTransfer extends AbstractReferenceCounted implements FileRegion { + private final ByteBuffer byteBufferHeader; + private final SelectMapedBufferResult selectMapedBufferResult; + private long transfered; // the bytes which was transfered already + + + public OneMessageTransfer(ByteBuffer byteBufferHeader, SelectMapedBufferResult selectMapedBufferResult) { + this.byteBufferHeader = byteBufferHeader; + this.selectMapedBufferResult = selectMapedBufferResult; + } + + + @Override + public long position() { + return this.byteBufferHeader.position() + this.selectMapedBufferResult.getByteBuffer().position(); + } + + + @Override + public long count() { + return this.byteBufferHeader.limit() + this.selectMapedBufferResult.getSize(); + } + + + @Override + public long transferTo(WritableByteChannel target, long position) throws IOException { + if (this.byteBufferHeader.hasRemaining()) { + transfered += target.write(this.byteBufferHeader); + return transfered; + } + else if (this.selectMapedBufferResult.getByteBuffer().hasRemaining()) { + transfered += target.write(this.selectMapedBufferResult.getByteBuffer()); + return transfered; + } + + return 0; + } + + + public void close() { + this.deallocate(); + } + + + @Override + protected void deallocate() { + this.selectMapedBufferResult.release(); + } + + + @Override + public long transfered() { + return transfered; + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/pagecache/QueryMessageTransfer.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/pagecache/QueryMessageTransfer.java new file mode 100644 index 000000000..b7f3c837c --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/pagecache/QueryMessageTransfer.java @@ -0,0 +1,97 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.broker.pagecache; + +import io.netty.channel.FileRegion; +import io.netty.util.AbstractReferenceCounted; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.WritableByteChannel; +import java.util.List; + +import com.alibaba.rocketmq.store.QueryMessageResult; + + +/** + * @author shijia.wxr + * @since 2013-7-26 + */ +public class QueryMessageTransfer extends AbstractReferenceCounted implements FileRegion { + private final ByteBuffer byteBufferHeader; + private final QueryMessageResult queryMessageResult; + private long transfered; // the bytes which was transfered already + + + public QueryMessageTransfer(ByteBuffer byteBufferHeader, QueryMessageResult queryMessageResult) { + this.byteBufferHeader = byteBufferHeader; + this.queryMessageResult = queryMessageResult; + } + + + @Override + public long position() { + int pos = byteBufferHeader.position(); + List messageBufferList = this.queryMessageResult.getMessageBufferList(); + for (ByteBuffer bb : messageBufferList) { + pos += bb.position(); + } + return pos; + } + + + @Override + public long count() { + return byteBufferHeader.limit() + this.queryMessageResult.getBufferTotalSize(); + } + + + @Override + public long transferTo(WritableByteChannel target, long position) throws IOException { + if (this.byteBufferHeader.hasRemaining()) { + transfered += target.write(this.byteBufferHeader); + return transfered; + } + else { + List messageBufferList = this.queryMessageResult.getMessageBufferList(); + for (ByteBuffer bb : messageBufferList) { + if (bb.hasRemaining()) { + transfered += target.write(bb); + return transfered; + } + } + } + + return 0; + } + + + public void close() { + this.deallocate(); + } + + + @Override + protected void deallocate() { + this.queryMessageResult.release(); + } + + + @Override + public long transfered() { + return transfered; + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/processor/AdminBrokerProcessor.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/processor/AdminBrokerProcessor.java new file mode 100644 index 000000000..466f47450 --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/processor/AdminBrokerProcessor.java @@ -0,0 +1,1232 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.broker.processor; + +import io.netty.channel.Channel; +import io.netty.channel.ChannelHandlerContext; + +import java.io.UnsupportedEncodingException; +import java.net.UnknownHostException; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Map; +import java.util.Properties; +import java.util.Set; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.rocketmq.broker.BrokerController; +import com.alibaba.rocketmq.broker.client.ClientChannelInfo; +import com.alibaba.rocketmq.broker.client.ConsumerGroupInfo; +import com.alibaba.rocketmq.common.MQVersion; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.TopicConfig; +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.admin.ConsumeStats; +import com.alibaba.rocketmq.common.admin.OffsetWrapper; +import com.alibaba.rocketmq.common.admin.TopicOffset; +import com.alibaba.rocketmq.common.admin.TopicStatsTable; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.common.message.MessageDecoder; +import com.alibaba.rocketmq.common.message.MessageId; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.common.protocol.RequestCode; +import com.alibaba.rocketmq.common.protocol.ResponseCode; +import com.alibaba.rocketmq.common.protocol.body.Connection; +import com.alibaba.rocketmq.common.protocol.body.ConsumerConnection; +import com.alibaba.rocketmq.common.protocol.body.GroupList; +import com.alibaba.rocketmq.common.protocol.body.KVTable; +import com.alibaba.rocketmq.common.protocol.body.LockBatchRequestBody; +import com.alibaba.rocketmq.common.protocol.body.LockBatchResponseBody; +import com.alibaba.rocketmq.common.protocol.body.ProducerConnection; +import com.alibaba.rocketmq.common.protocol.body.QueryConsumeTimeSpanBody; +import com.alibaba.rocketmq.common.protocol.body.QueryCorrectionOffsetBody; +import com.alibaba.rocketmq.common.protocol.body.QueueTimeSpan; +import com.alibaba.rocketmq.common.protocol.body.TopicList; +import com.alibaba.rocketmq.common.protocol.body.UnlockBatchRequestBody; +import com.alibaba.rocketmq.common.protocol.header.CloneGroupOffsetRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.ConsumeMessageDirectlyResultRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.CreateTopicRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.DeleteSubscriptionGroupRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.DeleteTopicRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.GetAllTopicConfigResponseHeader; +import com.alibaba.rocketmq.common.protocol.header.GetBrokerConfigResponseHeader; +import com.alibaba.rocketmq.common.protocol.header.GetConsumeStatsRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.GetConsumerConnectionListRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.GetConsumerRunningInfoRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.GetConsumerStatusRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.GetEarliestMsgStoretimeRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.GetEarliestMsgStoretimeResponseHeader; +import com.alibaba.rocketmq.common.protocol.header.GetMaxOffsetRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.GetMaxOffsetResponseHeader; +import com.alibaba.rocketmq.common.protocol.header.GetMinOffsetRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.GetMinOffsetResponseHeader; +import com.alibaba.rocketmq.common.protocol.header.GetProducerConnectionListRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.GetTopicStatsInfoRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.QueryConsumeTimeSpanRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.QueryCorrectionOffsetHeader; +import com.alibaba.rocketmq.common.protocol.header.QueryTopicConsumeByWhoRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.ResetOffsetRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.SearchOffsetRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.SearchOffsetResponseHeader; +import com.alibaba.rocketmq.common.protocol.header.filtersrv.RegisterFilterServerRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.filtersrv.RegisterFilterServerResponseHeader; +import com.alibaba.rocketmq.common.protocol.heartbeat.SubscriptionData; +import com.alibaba.rocketmq.common.subscription.SubscriptionGroupConfig; +import com.alibaba.rocketmq.remoting.common.RemotingHelper; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; +import com.alibaba.rocketmq.remoting.exception.RemotingTimeoutException; +import com.alibaba.rocketmq.remoting.netty.NettyRequestProcessor; +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; +import com.alibaba.rocketmq.store.DefaultMessageStore; +import com.alibaba.rocketmq.store.SelectMapedBufferResult; + + +/** + * 管理类请求处理 + * + * @author shijia.wxr + * @author manhong.yqd + * @since 2013-7-26 + */ +public class AdminBrokerProcessor implements NettyRequestProcessor { + private static final Logger log = LoggerFactory.getLogger(LoggerName.BrokerLoggerName); + private final BrokerController brokerController; + + + public AdminBrokerProcessor(final BrokerController brokerController) { + this.brokerController = brokerController; + } + + + @Override + public RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + switch (request.getCode()) { + // 更新创建Topic + case RequestCode.UPDATE_AND_CREATE_TOPIC: + return this.updateAndCreateTopic(ctx, request); + // 删除Topic + case RequestCode.DELETE_TOPIC_IN_BROKER: + return this.deleteTopic(ctx, request); + // 获取Topic配置 + case RequestCode.GET_ALL_TOPIC_CONFIG: + return this.getAllTopicConfig(ctx, request); + + // 更新Broker配置 TODO 可能存在并发问题 + case RequestCode.UPDATE_BROKER_CONFIG: + return this.updateBrokerConfig(ctx, request); + // 获取Broker配置 + case RequestCode.GET_BROKER_CONFIG: + return this.getBrokerConfig(ctx, request); + + // 根据时间查询Offset + case RequestCode.SEARCH_OFFSET_BY_TIMESTAMP: + return this.searchOffsetByTimestamp(ctx, request); + case RequestCode.GET_MAX_OFFSET: + return this.getMaxOffset(ctx, request); + case RequestCode.GET_MIN_OFFSET: + return this.getMinOffset(ctx, request); + case RequestCode.GET_EARLIEST_MSG_STORETIME: + return this.getEarliestMsgStoretime(ctx, request); + + // 获取Broker运行时信息 + case RequestCode.GET_BROKER_RUNTIME_INFO: + return this.getBrokerRuntimeInfo(ctx, request); + + // 锁队列与解锁队列 + case RequestCode.LOCK_BATCH_MQ: + return this.lockBatchMQ(ctx, request); + case RequestCode.UNLOCK_BATCH_MQ: + return this.unlockBatchMQ(ctx, request); + + // 订阅组配置 + case RequestCode.UPDATE_AND_CREATE_SUBSCRIPTIONGROUP: + return this.updateAndCreateSubscriptionGroup(ctx, request); + case RequestCode.GET_ALL_SUBSCRIPTIONGROUP_CONFIG: + return this.getAllSubscriptionGroup(ctx, request); + case RequestCode.DELETE_SUBSCRIPTIONGROUP: + return this.deleteSubscriptionGroup(ctx, request); + + // 统计信息，获取Topic统计信息 + case RequestCode.GET_TOPIC_STATS_INFO: + return this.getTopicStatsInfo(ctx, request); + + // Consumer连接管理 + case RequestCode.GET_CONSUMER_CONNECTION_LIST: + return this.getConsumerConnectionList(ctx, request); + // Producer连接管理 + case RequestCode.GET_PRODUCER_CONNECTION_LIST: + return this.getProducerConnectionList(ctx, request); + + // 查询消费进度，订阅组下的所有Topic + case RequestCode.GET_CONSUME_STATS: + return this.getConsumeStats(ctx, request); + case RequestCode.GET_ALL_CONSUMER_OFFSET: + return this.getAllConsumerOffset(ctx, request); + + // 定时进度 + case RequestCode.GET_ALL_DELAY_OFFSET: + return this.getAllDelayOffset(ctx, request); + + // 调用客户端重置 offset + case RequestCode.INVOKE_BROKER_TO_RESET_OFFSET: + return this.resetOffset(ctx, request); + + // 调用客户端订阅消息处理 + case RequestCode.INVOKE_BROKER_TO_GET_CONSUMER_STATUS: + return this.getConsumerStatus(ctx, request); + + // 查询Topic被哪些消费者消费 + case RequestCode.QUERY_TOPIC_CONSUME_BY_WHO: + return this.queryTopicConsumeByWho(ctx, request); + + case RequestCode.REGISTER_FILTER_SERVER: + return this.registerFilterServer(ctx, request); + // 根据 topic 和 group 获取消息的时间跨度 + case RequestCode.QUERY_CONSUME_TIME_SPAN: + return this.queryConsumeTimeSpan(ctx, request); + case RequestCode.GET_SYSTEM_TOPIC_LIST_FROM_BROKER: + return this.getSystemTopicListFromBroker(ctx, request); + + // 删除失效队列 + case RequestCode.CLEAN_EXPIRED_CONSUMEQUEUE: + return this.cleanExpiredConsumeQueue(); + + case RequestCode.GET_CONSUMER_RUNNING_INFO: + return this.getConsumerRunningInfo(ctx, request); + + // 查找被修正 offset (转发组件） + case RequestCode.QUERY_CORRECTION_OFFSET: + return this.queryCorrectionOffset(ctx, request); + + case RequestCode.CONSUME_MESSAGE_DIRECTLY: + return this.consumeMessageDirectly(ctx, request); + case RequestCode.CLONE_GROUP_OFFSET: + return this.cloneGroupOffset(ctx, request); + + default: + break; + } + + return null; + } + + + private RemotingCommand callConsumer(// + final int requestCode,// + final RemotingCommand request, // + final String consumerGroup,// + final String clientId) throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + ClientChannelInfo clientChannelInfo = + this.brokerController.getConsumerManager().findChannel(consumerGroup, clientId); + + if (null == clientChannelInfo) { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark(String.format("The Consumer <%s> <%s> not online", consumerGroup, clientId)); + return response; + } + + if (clientChannelInfo.getVersion() < MQVersion.Version.V3_1_8_SNAPSHOT.ordinal()) { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark(String.format( + "The Consumer <%s> Version <%s> too low to finish, please upgrade it to V3_1_8_SNAPSHOT", // + clientId,// + MQVersion.getVersionDesc(clientChannelInfo.getVersion()))); + return response; + } + + try { + RemotingCommand newRequest = RemotingCommand.createRequestCommand(requestCode, null); + newRequest.setExtFields(request.getExtFields()); + newRequest.setBody(request.getBody()); + + RemotingCommand consumerResponse = + this.brokerController.getBroker2Client().callClient(clientChannelInfo.getChannel(), + newRequest); + return consumerResponse; + } + catch (RemotingTimeoutException e) { + response.setCode(ResponseCode.CONSUME_MSG_TIMEOUT); + response.setRemark(String.format("consumer <%s> <%s> Timeout: %s", consumerGroup, clientId, + RemotingHelper.exceptionSimpleDesc(e))); + return response; + } + catch (Exception e) { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark(String.format("invoke consumer <%s> <%s> Exception: %s", consumerGroup, + clientId, RemotingHelper.exceptionSimpleDesc(e))); + return response; + } + } + + + private RemotingCommand consumeMessageDirectly(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + final ConsumeMessageDirectlyResultRequestHeader requestHeader = + (ConsumeMessageDirectlyResultRequestHeader) request + .decodeCommandCustomHeader(ConsumeMessageDirectlyResultRequestHeader.class); + + request.getExtFields().put("brokerName", this.brokerController.getBrokerConfig().getBrokerName()); + SelectMapedBufferResult selectMapedBufferResult = null; + try { + MessageId messageId = MessageDecoder.decodeMessageId(requestHeader.getMsgId()); + selectMapedBufferResult = + this.brokerController.getMessageStore().selectOneMessageByOffset(messageId.getOffset()); + + byte[] body = new byte[selectMapedBufferResult.getSize()]; + selectMapedBufferResult.getByteBuffer().get(body); + request.setBody(body); + } + catch (UnknownHostException e) { + } + finally { + if (selectMapedBufferResult != null) { + selectMapedBufferResult.release(); + } + } + + return this.callConsumer(RequestCode.CONSUME_MESSAGE_DIRECTLY, request, + requestHeader.getConsumerGroup(), requestHeader.getClientId()); + } + + + /** + * 调用Consumer，获取Consumer内存数据结构，为监控以及定位问题 + */ + private RemotingCommand getConsumerRunningInfo(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + final GetConsumerRunningInfoRequestHeader requestHeader = + (GetConsumerRunningInfoRequestHeader) request + .decodeCommandCustomHeader(GetConsumerRunningInfoRequestHeader.class); + + return this.callConsumer(RequestCode.GET_CONSUMER_RUNNING_INFO, request, + requestHeader.getConsumerGroup(), requestHeader.getClientId()); + } + + + public RemotingCommand cleanExpiredConsumeQueue() { + log.warn("invoke cleanExpiredConsumeQueue start."); + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + brokerController.getMessageStore().cleanExpiredConsumerQueue(); + log.warn("invoke cleanExpiredConsumeQueue end."); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + + private RemotingCommand registerFilterServer(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + final RemotingCommand response = + RemotingCommand.createResponseCommand(RegisterFilterServerResponseHeader.class); + final RegisterFilterServerResponseHeader responseHeader = + (RegisterFilterServerResponseHeader) response.readCustomHeader(); + final RegisterFilterServerRequestHeader requestHeader = + (RegisterFilterServerRequestHeader) request + .decodeCommandCustomHeader(RegisterFilterServerRequestHeader.class); + + this.brokerController.getFilterServerManager().registerFilterServer(ctx.channel(), + requestHeader.getFilterServerAddr()); + + responseHeader.setBrokerId(this.brokerController.getBrokerConfig().getBrokerId()); + responseHeader.setBrokerName(this.brokerController.getBrokerConfig().getBrokerName()); + + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + + private RemotingCommand getConsumeStats(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + final GetConsumeStatsRequestHeader requestHeader = + (GetConsumeStatsRequestHeader) request + .decodeCommandCustomHeader(GetConsumeStatsRequestHeader.class); + + ConsumeStats consumeStats = new ConsumeStats(); + + Set topics = + this.brokerController.getConsumerOffsetManager().whichTopicByConsumer( + requestHeader.getConsumerGroup()); + + for (String topic : topics) { + TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(topic); + if (null == topicConfig) { + log.warn("consumeStats, topic config not exist, {}", topic); + continue; + } + + /** + * Consumer不在线的时候，也允许查询消费进度 + */ + { + SubscriptionData findSubscriptionData = + this.brokerController.getConsumerManager().findSubscriptionData( + requestHeader.getConsumerGroup(), topic); + // 如果Consumer在线，而且这个topic没有被订阅，那么就跳过 + if (null == findSubscriptionData // + && this.brokerController.getConsumerManager().findSubscriptionDataCount( + requestHeader.getConsumerGroup()) > 0) { + log.warn("consumeStats, the consumer group[{}], topic[{}] not exist", + requestHeader.getConsumerGroup(), topic); + continue; + } + } + + for (int i = 0; i < topicConfig.getWriteQueueNums(); i++) { + MessageQueue mq = new MessageQueue(); + mq.setTopic(topic); + mq.setBrokerName(this.brokerController.getBrokerConfig().getBrokerName()); + mq.setQueueId(i); + + OffsetWrapper offsetWrapper = new OffsetWrapper(); + + long brokerOffset = this.brokerController.getMessageStore().getMaxOffsetInQuque(topic, i); + if (brokerOffset < 0) + brokerOffset = 0; + + long consumerOffset = this.brokerController.getConsumerOffsetManager().queryOffset(// + requestHeader.getConsumerGroup(),// + topic,// + i); + if (consumerOffset < 0) + consumerOffset = 0; + + offsetWrapper.setBrokerOffset(brokerOffset); + offsetWrapper.setConsumerOffset(consumerOffset); + + // 查询消费者最后一条消息对应的时间戳 + long timeOffset = consumerOffset - 1; + if (timeOffset >= 0) { + long lastTimestamp = + this.brokerController.getMessageStore().getMessageStoreTimeStamp(topic, i, + timeOffset); + if (lastTimestamp > 0) { + offsetWrapper.setLastTimestamp(lastTimestamp); + } + } + + consumeStats.getOffsetTable().put(mq, offsetWrapper); + } + + long consumeTps = + (long) this.brokerController.getBrokerStatsManager().tpsGroupGetNums( + requestHeader.getConsumerGroup(), topic); + + consumeTps += consumeStats.getConsumeTps(); + consumeStats.setConsumeTps(consumeTps); + } + + byte[] body = consumeStats.encode(); + response.setBody(body); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + + private RemotingCommand getProducerConnectionList(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + final GetProducerConnectionListRequestHeader requestHeader = + (GetProducerConnectionListRequestHeader) request + .decodeCommandCustomHeader(GetProducerConnectionListRequestHeader.class); + + ProducerConnection bodydata = new ProducerConnection(); + HashMap channelInfoHashMap = + this.brokerController.getProducerManager().getGroupChannelTable() + .get(requestHeader.getProducerGroup()); + if (channelInfoHashMap != null) { + Iterator> it = channelInfoHashMap.entrySet().iterator(); + while (it.hasNext()) { + ClientChannelInfo info = it.next().getValue(); + Connection connection = new Connection(); + connection.setClientId(info.getClientId()); + connection.setLanguage(info.getLanguage()); + connection.setVersion(info.getVersion()); + connection.setClientAddr(RemotingHelper.parseChannelRemoteAddr(info.getChannel())); + + bodydata.getConnectionSet().add(connection); + } + + byte[] body = bodydata.encode(); + response.setBody(body); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("the producer group[" + requestHeader.getProducerGroup() + "] not exist"); + return response; + } + + + private RemotingCommand getConsumerConnectionList(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + final GetConsumerConnectionListRequestHeader requestHeader = + (GetConsumerConnectionListRequestHeader) request + .decodeCommandCustomHeader(GetConsumerConnectionListRequestHeader.class); + + ConsumerGroupInfo consumerGroupInfo = + this.brokerController.getConsumerManager().getConsumerGroupInfo( + requestHeader.getConsumerGroup()); + if (consumerGroupInfo != null) { + ConsumerConnection bodydata = new ConsumerConnection(); + bodydata.setConsumeFromWhere(consumerGroupInfo.getConsumeFromWhere()); + bodydata.setConsumeType(consumerGroupInfo.getConsumeType()); + bodydata.setMessageModel(consumerGroupInfo.getMessageModel()); + bodydata.getSubscriptionTable().putAll(consumerGroupInfo.getSubscriptionTable()); + + Iterator> it = + consumerGroupInfo.getChannelInfoTable().entrySet().iterator(); + while (it.hasNext()) { + ClientChannelInfo info = it.next().getValue(); + Connection connection = new Connection(); + connection.setClientId(info.getClientId()); + connection.setLanguage(info.getLanguage()); + connection.setVersion(info.getVersion()); + connection.setClientAddr(RemotingHelper.parseChannelRemoteAddr(info.getChannel())); + + bodydata.getConnectionSet().add(connection); + } + + byte[] body = bodydata.encode(); + response.setBody(body); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + + return response; + } + + response.setCode(ResponseCode.SUBSCRIPTION_GROUP_NOT_EXIST); + response.setRemark("the consumer group[" + requestHeader.getConsumerGroup() + "] not online"); + return response; + } + + + private RemotingCommand getTopicStatsInfo(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + final GetTopicStatsInfoRequestHeader requestHeader = + (GetTopicStatsInfoRequestHeader) request + .decodeCommandCustomHeader(GetTopicStatsInfoRequestHeader.class); + + final String topic = requestHeader.getTopic(); + TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(topic); + if (null == topicConfig) { + response.setCode(ResponseCode.TOPIC_NOT_EXIST); + response.setRemark("topic[" + topic + "] not exist"); + return response; + } + + TopicStatsTable topicStatsTable = new TopicStatsTable(); + for (int i = 0; i < topicConfig.getWriteQueueNums(); i++) { + MessageQueue mq = new MessageQueue(); + mq.setTopic(topic); + mq.setBrokerName(this.brokerController.getBrokerConfig().getBrokerName()); + mq.setQueueId(i); + + TopicOffset topicOffset = new TopicOffset(); + long min = this.brokerController.getMessageStore().getMinOffsetInQuque(topic, i); + if (min < 0) + min = 0; + + long max = this.brokerController.getMessageStore().getMaxOffsetInQuque(topic, i); + if (max < 0) + max = 0; + + long timestamp = 0; + if (max > 0) { + timestamp = + this.brokerController.getMessageStore().getMessageStoreTimeStamp(topic, i, (max - 1)); + } + + topicOffset.setMinOffset(min); + topicOffset.setMaxOffset(max); + topicOffset.setLastUpdateTimestamp(timestamp); + + topicStatsTable.getOffsetTable().put(mq, topicOffset); + } + + byte[] body = topicStatsTable.encode(); + response.setBody(body); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + + private RemotingCommand updateAndCreateSubscriptionGroup(ChannelHandlerContext ctx, + RemotingCommand request) throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + + log.info("updateAndCreateSubscriptionGroup called by {}", + RemotingHelper.parseChannelRemoteAddr(ctx.channel())); + + SubscriptionGroupConfig config = + RemotingSerializable.decode(request.getBody(), SubscriptionGroupConfig.class); + if (config != null) { + this.brokerController.getSubscriptionGroupManager().updateSubscriptionGroupConfig(config); + } + + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + + private RemotingCommand getAllSubscriptionGroup(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + String content = this.brokerController.getSubscriptionGroupManager().encode(); + if (content != null && content.length() > 0) { + try { + response.setBody(content.getBytes(MixAll.DEFAULT_CHARSET)); + } + catch (UnsupportedEncodingException e) { + log.error("", e); + + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("UnsupportedEncodingException " + e); + return response; + } + } + else { + log.error("No subscription group in this broker, client: " + ctx.channel().remoteAddress()); + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("No subscription group in this broker"); + return response; + } + + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + + return response; + } + + + private RemotingCommand lockBatchMQ(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + LockBatchRequestBody requestBody = + LockBatchRequestBody.decode(request.getBody(), LockBatchRequestBody.class); + + Set lockOKMQSet = this.brokerController.getRebalanceLockManager().tryLockBatch(// + requestBody.getConsumerGroup(),// + requestBody.getMqSet(),// + requestBody.getClientId()); + + LockBatchResponseBody responseBody = new LockBatchResponseBody(); + responseBody.setLockOKMQSet(lockOKMQSet); + + response.setBody(responseBody.encode()); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + + private RemotingCommand unlockBatchMQ(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + UnlockBatchRequestBody requestBody = + UnlockBatchRequestBody.decode(request.getBody(), UnlockBatchRequestBody.class); + + this.brokerController.getRebalanceLockManager().unlockBatch(// + requestBody.getConsumerGroup(),// + requestBody.getMqSet(),// + requestBody.getClientId()); + + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + + private RemotingCommand updateAndCreateTopic(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + final CreateTopicRequestHeader requestHeader = + (CreateTopicRequestHeader) request.decodeCommandCustomHeader(CreateTopicRequestHeader.class); + log.info("updateAndCreateTopic called by {}", RemotingHelper.parseChannelRemoteAddr(ctx.channel())); + + // Topic名字是否与保留字段冲突 + if (requestHeader.getTopic().equals(this.brokerController.getBrokerConfig().getBrokerClusterName())) { + String errorMsg = + "the topic[" + requestHeader.getTopic() + "] is conflict with system reserved words."; + log.warn(errorMsg); + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark(errorMsg); + return response; + } + + TopicConfig topicConfig = new TopicConfig(requestHeader.getTopic()); + topicConfig.setReadQueueNums(requestHeader.getReadQueueNums()); + topicConfig.setWriteQueueNums(requestHeader.getWriteQueueNums()); + topicConfig.setTopicFilterType(requestHeader.getTopicFilterTypeEnum()); + topicConfig.setPerm(requestHeader.getPerm()); + topicConfig.setTopicSysFlag(requestHeader.getTopicSysFlag() == null ? 0 : requestHeader + .getTopicSysFlag()); + + this.brokerController.getTopicConfigManager().updateTopicConfig(topicConfig); + + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + + private RemotingCommand deleteTopic(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + DeleteTopicRequestHeader requestHeader = + (DeleteTopicRequestHeader) request.decodeCommandCustomHeader(DeleteTopicRequestHeader.class); + + log.info("deleteTopic called by {}", RemotingHelper.parseChannelRemoteAddr(ctx.channel())); + + this.brokerController.getTopicConfigManager().deleteTopicConfig(requestHeader.getTopic()); + this.brokerController.addDeleteTopicTask(); + + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + + private RemotingCommand getAllTopicConfig(ChannelHandlerContext ctx, RemotingCommand request) { + final RemotingCommand response = + RemotingCommand.createResponseCommand(GetAllTopicConfigResponseHeader.class); + // final GetAllTopicConfigResponseHeader responseHeader = + // (GetAllTopicConfigResponseHeader) response.readCustomHeader(); + + String content = this.brokerController.getTopicConfigManager().encode(); + if (content != null && content.length() > 0) { + try { + response.setBody(content.getBytes(MixAll.DEFAULT_CHARSET)); + } + catch (UnsupportedEncodingException e) { + log.error("", e); + + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("UnsupportedEncodingException " + e); + return response; + } + } + else { + log.error("No topic in this broker, client: " + ctx.channel().remoteAddress()); + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("No topic in this broker"); + return response; + } + + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + + return response; + } + + + private RemotingCommand updateBrokerConfig(ChannelHandlerContext ctx, RemotingCommand request) { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + + log.info("updateBrokerConfig called by {}", RemotingHelper.parseChannelRemoteAddr(ctx.channel())); + + byte[] body = request.getBody(); + if (body != null) { + try { + String bodyStr = new String(body, MixAll.DEFAULT_CHARSET); + Properties properties = MixAll.string2Properties(bodyStr); + if (properties != null) { + log.info("updateBrokerConfig, new config: " + properties + " client: " + + ctx.channel().remoteAddress()); + this.brokerController.updateAllConfig(properties); + } + else { + log.error("string2Properties error"); + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("string2Properties error"); + return response; + } + } + catch (UnsupportedEncodingException e) { + log.error("", e); + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("UnsupportedEncodingException " + e); + return response; + } + } + + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + + private RemotingCommand getBrokerConfig(ChannelHandlerContext ctx, RemotingCommand request) { + + final RemotingCommand response = + RemotingCommand.createResponseCommand(GetBrokerConfigResponseHeader.class); + final GetBrokerConfigResponseHeader responseHeader = + (GetBrokerConfigResponseHeader) response.readCustomHeader(); + + String content = this.brokerController.encodeAllConfig(); + if (content != null && content.length() > 0) { + try { + response.setBody(content.getBytes(MixAll.DEFAULT_CHARSET)); + } + catch (UnsupportedEncodingException e) { + log.error("", e); + + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("UnsupportedEncodingException " + e); + return response; + } + } + + responseHeader.setVersion(this.brokerController.getConfigDataVersion()); + + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + + private RemotingCommand searchOffsetByTimestamp(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + final RemotingCommand response = + RemotingCommand.createResponseCommand(SearchOffsetResponseHeader.class); + final SearchOffsetResponseHeader responseHeader = + (SearchOffsetResponseHeader) response.readCustomHeader(); + final SearchOffsetRequestHeader requestHeader = + (SearchOffsetRequestHeader) request + .decodeCommandCustomHeader(SearchOffsetRequestHeader.class); + + long offset = + this.brokerController.getMessageStore().getOffsetInQueueByTime(requestHeader.getTopic(), + requestHeader.getQueueId(), requestHeader.getTimestamp()); + + responseHeader.setOffset(offset); + + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + + private RemotingCommand getMaxOffset(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + final RemotingCommand response = + RemotingCommand.createResponseCommand(GetMaxOffsetResponseHeader.class); + final GetMaxOffsetResponseHeader responseHeader = + (GetMaxOffsetResponseHeader) response.readCustomHeader(); + final GetMaxOffsetRequestHeader requestHeader = + (GetMaxOffsetRequestHeader) request + .decodeCommandCustomHeader(GetMaxOffsetRequestHeader.class); + + long offset = + this.brokerController.getMessageStore().getMaxOffsetInQuque(requestHeader.getTopic(), + requestHeader.getQueueId()); + + responseHeader.setOffset(offset); + + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + + private RemotingCommand getMinOffset(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + final RemotingCommand response = + RemotingCommand.createResponseCommand(GetMinOffsetResponseHeader.class); + final GetMinOffsetResponseHeader responseHeader = + (GetMinOffsetResponseHeader) response.readCustomHeader(); + final GetMinOffsetRequestHeader requestHeader = + (GetMinOffsetRequestHeader) request + .decodeCommandCustomHeader(GetMinOffsetRequestHeader.class); + + long offset = + this.brokerController.getMessageStore().getMinOffsetInQuque(requestHeader.getTopic(), + requestHeader.getQueueId()); + + responseHeader.setOffset(offset); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + + private RemotingCommand getEarliestMsgStoretime(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + final RemotingCommand response = + RemotingCommand.createResponseCommand(GetEarliestMsgStoretimeResponseHeader.class); + final GetEarliestMsgStoretimeResponseHeader responseHeader = + (GetEarliestMsgStoretimeResponseHeader) response.readCustomHeader(); + final GetEarliestMsgStoretimeRequestHeader requestHeader = + (GetEarliestMsgStoretimeRequestHeader) request + .decodeCommandCustomHeader(GetEarliestMsgStoretimeRequestHeader.class); + + long timestamp = + this.brokerController.getMessageStore().getEarliestMessageTime(requestHeader.getTopic(), + requestHeader.getQueueId()); + + responseHeader.setTimestamp(timestamp); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + + private HashMap prepareRuntimeInfo() { + HashMap runtimeInfo = this.brokerController.getMessageStore().getRuntimeInfo(); + runtimeInfo.put("brokerVersionDesc", MQVersion.getVersionDesc(MQVersion.CurrentVersion)); + runtimeInfo.put("brokerVersion", String.valueOf(MQVersion.CurrentVersion)); + + runtimeInfo.put("msgPutTotalYesterdayMorning", + String.valueOf(this.brokerController.getBrokerStats().getMsgPutTotalYesterdayMorning())); + runtimeInfo.put("msgPutTotalTodayMorning", + String.valueOf(this.brokerController.getBrokerStats().getMsgPutTotalTodayMorning())); + runtimeInfo.put("msgPutTotalTodayNow", + String.valueOf(this.brokerController.getBrokerStats().getMsgPutTotalTodayNow())); + + runtimeInfo.put("msgGetTotalYesterdayMorning", + String.valueOf(this.brokerController.getBrokerStats().getMsgGetTotalYesterdayMorning())); + runtimeInfo.put("msgGetTotalTodayMorning", + String.valueOf(this.brokerController.getBrokerStats().getMsgGetTotalTodayMorning())); + runtimeInfo.put("msgGetTotalTodayNow", + String.valueOf(this.brokerController.getBrokerStats().getMsgGetTotalTodayNow())); + + runtimeInfo.put("sendThreadPoolQueueSize", + String.valueOf(this.brokerController.getSendThreadPoolQueue().size())); + + runtimeInfo.put("sendThreadPoolQueueCapacity", + String.valueOf(this.brokerController.getBrokerConfig().getSendThreadPoolQueueCapacity())); + + return runtimeInfo; + } + + + private RemotingCommand getBrokerRuntimeInfo(ChannelHandlerContext ctx, RemotingCommand request) { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + + HashMap runtimeInfo = this.prepareRuntimeInfo(); + KVTable kvTable = new KVTable(); + kvTable.setTable(runtimeInfo); + + byte[] body = kvTable.encode(); + response.setBody(body); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + + private RemotingCommand getAllConsumerOffset(ChannelHandlerContext ctx, RemotingCommand request) { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + + String content = this.brokerController.getConsumerOffsetManager().encode(); + if (content != null && content.length() > 0) { + try { + response.setBody(content.getBytes(MixAll.DEFAULT_CHARSET)); + } + catch (UnsupportedEncodingException e) { + log.error("get all consumer offset from master error.", e); + + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("UnsupportedEncodingException " + e); + return response; + } + } + else { + log.error("No consumer offset in this broker, client: " + ctx.channel().remoteAddress()); + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("No consumer offset in this broker"); + return response; + } + + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + + return response; + } + + + private RemotingCommand getAllDelayOffset(ChannelHandlerContext ctx, RemotingCommand request) { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + + String content = + ((DefaultMessageStore) this.brokerController.getMessageStore()).getScheduleMessageService() + .encode(); + if (content != null && content.length() > 0) { + try { + response.setBody(content.getBytes(MixAll.DEFAULT_CHARSET)); + } + catch (UnsupportedEncodingException e) { + log.error("get all delay offset from master error.", e); + + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("UnsupportedEncodingException " + e); + return response; + } + } + else { + log.error("No delay offset in this broker, client: " + ctx.channel().remoteAddress()); + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("No delay offset in this broker"); + return response; + } + + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + + return response; + } + + + private RemotingCommand deleteSubscriptionGroup(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + DeleteSubscriptionGroupRequestHeader requestHeader = + (DeleteSubscriptionGroupRequestHeader) request + .decodeCommandCustomHeader(DeleteSubscriptionGroupRequestHeader.class); + + log.info("deleteSubscriptionGroup called by {}", RemotingHelper.parseChannelRemoteAddr(ctx.channel())); + + this.brokerController.getSubscriptionGroupManager().deleteSubscriptionGroupConfig( + requestHeader.getGroupName()); + + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + + public RemotingCommand resetOffset(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + final ResetOffsetRequestHeader requestHeader = + (ResetOffsetRequestHeader) request.decodeCommandCustomHeader(ResetOffsetRequestHeader.class); + log.info("[reset-offset] reset offset started by {}. topic={}, group={}, timestamp={}, isForce={}", + new Object[] { RemotingHelper.parseChannelRemoteAddr(ctx.channel()), requestHeader.getTopic(), + requestHeader.getGroup(), requestHeader.getTimestamp(), requestHeader.isForce() }); + return this.brokerController.getBroker2Client().resetOffset(requestHeader.getTopic(), + requestHeader.getGroup(), requestHeader.getTimestamp(), requestHeader.isForce()); + } + + + public RemotingCommand getConsumerStatus(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + final GetConsumerStatusRequestHeader requestHeader = + (GetConsumerStatusRequestHeader) request + .decodeCommandCustomHeader(GetConsumerStatusRequestHeader.class); + + log.info("[get-consumer-status] get consumer status by {}. topic={}, group={}", + new Object[] { RemotingHelper.parseChannelRemoteAddr(ctx.channel()), requestHeader.getTopic(), + requestHeader.getGroup() }); + + return this.brokerController.getBroker2Client().getConsumeStatus(requestHeader.getTopic(), + requestHeader.getGroup(), requestHeader.getClientAddr()); + } + + + private RemotingCommand queryTopicConsumeByWho(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + QueryTopicConsumeByWhoRequestHeader requestHeader = + (QueryTopicConsumeByWhoRequestHeader) request + .decodeCommandCustomHeader(QueryTopicConsumeByWhoRequestHeader.class); + + // 从订阅关系查询topic被谁消费，只查询在线 + HashSet groups = + this.brokerController.getConsumerManager().queryTopicConsumeByWho(requestHeader.getTopic()); + // 从Offset持久化查询topic被谁消费，离线和在线都会查询 + Set groupInOffset = + this.brokerController.getConsumerOffsetManager().whichGroupByTopic(requestHeader.getTopic()); + if (groupInOffset != null && !groupInOffset.isEmpty()) { + groups.addAll(groupInOffset); + } + + GroupList groupList = new GroupList(); + groupList.setGroupList(groups); + byte[] body = groupList.encode(); + + response.setBody(body); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + + private RemotingCommand queryConsumeTimeSpan(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + QueryConsumeTimeSpanRequestHeader requestHeader = + (QueryConsumeTimeSpanRequestHeader) request + .decodeCommandCustomHeader(QueryConsumeTimeSpanRequestHeader.class); + + final String topic = requestHeader.getTopic(); + TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(topic); + if (null == topicConfig) { + response.setCode(ResponseCode.TOPIC_NOT_EXIST); + response.setRemark("topic[" + topic + "] not exist"); + return response; + } + + Set timeSpanSet = new HashSet(); + for (int i = 0; i < topicConfig.getWriteQueueNums(); i++) { + QueueTimeSpan timeSpan = new QueueTimeSpan(); + MessageQueue mq = new MessageQueue(); + mq.setTopic(topic); + mq.setBrokerName(this.brokerController.getBrokerConfig().getBrokerName()); + mq.setQueueId(i); + timeSpan.setMessageQueue(mq); + + long minTime = this.brokerController.getMessageStore().getEarliestMessageTime(topic, i); + timeSpan.setMinTimeStamp(minTime); + + long max = this.brokerController.getMessageStore().getMaxOffsetInQuque(topic, i); + long maxTime = + this.brokerController.getMessageStore().getMessageStoreTimeStamp(topic, i, (max - 1)); + timeSpan.setMaxTimeStamp(maxTime); + + long consumeTime; + long consumerOffset = this.brokerController.getConsumerOffsetManager().queryOffset(// + requestHeader.getGroup(), topic, i); + if (consumerOffset > 0) { + consumeTime = + this.brokerController.getMessageStore().getMessageStoreTimeStamp(topic, i, + consumerOffset); + } + else { + consumeTime = minTime; + } + timeSpan.setConsumeTimeStamp(consumeTime); + timeSpanSet.add(timeSpan); + } + + QueryConsumeTimeSpanBody queryConsumeTimeSpanBody = new QueryConsumeTimeSpanBody(); + queryConsumeTimeSpanBody.setConsumeTimeSpanSet(timeSpanSet); + response.setBody(queryConsumeTimeSpanBody.encode()); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + + private RemotingCommand getSystemTopicListFromBroker(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + + Set topics = this.brokerController.getTopicConfigManager().getSystemTopic(); + TopicList topicList = new TopicList(); + topicList.setTopicList(topics); + response.setBody(topicList.encode()); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + + private RemotingCommand queryCorrectionOffset(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + QueryCorrectionOffsetHeader requestHeader = + (QueryCorrectionOffsetHeader) request + .decodeCommandCustomHeader(QueryCorrectionOffsetHeader.class); + + Map correctionOffset = + this.brokerController.getConsumerOffsetManager().queryMinOffsetInAllGroup( + requestHeader.getTopic(), requestHeader.getFilterGroups()); + + Map compareOffset = + this.brokerController.getConsumerOffsetManager().queryOffset(requestHeader.getTopic(), + requestHeader.getCompareGroup()); + + if (compareOffset != null && !compareOffset.isEmpty()) { + for (Integer queueId : compareOffset.keySet()) { + correctionOffset.put(queueId, + correctionOffset.get(queueId) > compareOffset.get(queueId) ? Long.MAX_VALUE + : correctionOffset.get(queueId)); + } + } + + QueryCorrectionOffsetBody body = new QueryCorrectionOffsetBody(); + body.setCorrectionOffsets(correctionOffset); + response.setBody(body.encode()); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + + private RemotingCommand cloneGroupOffset(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + CloneGroupOffsetRequestHeader requestHeader = + (CloneGroupOffsetRequestHeader) request + .decodeCommandCustomHeader(CloneGroupOffsetRequestHeader.class); + + Set topics; + if (UtilAll.isBlank(requestHeader.getTopic())) { + topics = + this.brokerController.getConsumerOffsetManager().whichTopicByConsumer( + requestHeader.getSrcGroup()); + } + else { + topics = new HashSet(); + topics.add(requestHeader.getTopic()); + } + + for (String topic : topics) { + TopicConfig topicConfig = this.brokerController.getTopicConfigManager().selectTopicConfig(topic); + if (null == topicConfig) { + log.warn("[cloneGroupOffset], topic config not exist, {}", topic); + continue; + } + + /** + * Consumer不在线的时候，也允许查询消费进度 + */ + if (!requestHeader.isOffline()) { + // 如果Consumer在线，而且这个topic没有被订阅，那么就跳过 + SubscriptionData findSubscriptionData = + this.brokerController.getConsumerManager().findSubscriptionData( + requestHeader.getSrcGroup(), topic); + if (this.brokerController.getConsumerManager().findSubscriptionDataCount( + requestHeader.getSrcGroup()) > 0 + && findSubscriptionData == null) { + log.warn("[cloneGroupOffset], the consumer group[{}], topic[{}] not exist", + requestHeader.getSrcGroup(), topic); + continue; + } + } + + this.brokerController.getConsumerOffsetManager().cloneOffset(requestHeader.getSrcGroup(), + requestHeader.getDestGroup(), requestHeader.getTopic()); + } + + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/processor/ClientManageProcessor.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/processor/ClientManageProcessor.java new file mode 100644 index 000000000..9608de3c3 --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/processor/ClientManageProcessor.java @@ -0,0 +1,350 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.broker.processor; + +import io.netty.channel.ChannelHandlerContext; + +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.util.List; +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.rocketmq.broker.BrokerController; +import com.alibaba.rocketmq.broker.client.ClientChannelInfo; +import com.alibaba.rocketmq.broker.client.ConsumerGroupInfo; +import com.alibaba.rocketmq.broker.mqtrace.ConsumeMessageContext; +import com.alibaba.rocketmq.broker.mqtrace.ConsumeMessageHook; +import com.alibaba.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.common.constant.PermName; +import com.alibaba.rocketmq.common.protocol.RequestCode; +import com.alibaba.rocketmq.common.protocol.ResponseCode; +import com.alibaba.rocketmq.common.protocol.header.GetConsumerListByGroupRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.GetConsumerListByGroupResponseBody; +import com.alibaba.rocketmq.common.protocol.header.GetConsumerListByGroupResponseHeader; +import com.alibaba.rocketmq.common.protocol.header.QueryConsumerOffsetRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.QueryConsumerOffsetResponseHeader; +import com.alibaba.rocketmq.common.protocol.header.UnregisterClientRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.UnregisterClientResponseHeader; +import com.alibaba.rocketmq.common.protocol.header.UpdateConsumerOffsetRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.UpdateConsumerOffsetResponseHeader; +import com.alibaba.rocketmq.common.protocol.heartbeat.ConsumerData; +import com.alibaba.rocketmq.common.protocol.heartbeat.HeartbeatData; +import com.alibaba.rocketmq.common.protocol.heartbeat.ProducerData; +import com.alibaba.rocketmq.common.subscription.SubscriptionGroupConfig; +import com.alibaba.rocketmq.common.sysflag.TopicSysFlag; +import com.alibaba.rocketmq.remoting.common.RemotingHelper; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; +import com.alibaba.rocketmq.remoting.netty.NettyRequestProcessor; +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; + + +/** + * Client注册与注销管理 + * + * @author shijia.wxr + * @since 2013-7-26 + */ +public class ClientManageProcessor implements NettyRequestProcessor { + private static final Logger log = LoggerFactory.getLogger(LoggerName.BrokerLoggerName); + + private final BrokerController brokerController; + + + public ClientManageProcessor(final BrokerController brokerController) { + this.brokerController = brokerController; + } + + + @Override + public RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + switch (request.getCode()) { + case RequestCode.HEART_BEAT: + return this.heartBeat(ctx, request); + case RequestCode.UNREGISTER_CLIENT: + return this.unregisterClient(ctx, request); + case RequestCode.GET_CONSUMER_LIST_BY_GROUP: + return this.getConsumerListByGroup(ctx, request); + // 更新Consumer Offset + case RequestCode.UPDATE_CONSUMER_OFFSET: + return this.updateConsumerOffset(ctx, request); + case RequestCode.QUERY_CONSUMER_OFFSET: + return this.queryConsumerOffset(ctx, request); + default: + break; + } + return null; + } + + /** + * 消费每条消息会回调 + */ + private List consumeMessageHookList; + + + public boolean hasConsumeMessageHook() { + return consumeMessageHookList != null && !this.consumeMessageHookList.isEmpty(); + } + + + public void registerConsumeMessageHook(List consumeMessageHookList) { + this.consumeMessageHookList = consumeMessageHookList; + } + + + public void executeConsumeMessageHookAfter(final ConsumeMessageContext context) { + if (hasConsumeMessageHook()) { + for (ConsumeMessageHook hook : this.consumeMessageHookList) { + try { + hook.consumeMessageAfter(context); + } + catch (Throwable e) { + } + } + } + } + + + private RemotingCommand updateConsumerOffset(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + final RemotingCommand response = + RemotingCommand.createResponseCommand(UpdateConsumerOffsetResponseHeader.class); + // final UpdateConsumerOffsetResponseHeader responseHeader = + // (UpdateConsumerOffsetResponseHeader) response.readCustomHeader(); + final UpdateConsumerOffsetRequestHeader requestHeader = + (UpdateConsumerOffsetRequestHeader) request + .decodeCommandCustomHeader(UpdateConsumerOffsetRequestHeader.class); + + // 消息轨迹：记录已经消费成功并提交 offset 的消息记录 + if (this.hasConsumeMessageHook()) { + // 执行hook + ConsumeMessageContext context = new ConsumeMessageContext(); + context.setConsumerGroup(requestHeader.getConsumerGroup()); + context.setTopic(requestHeader.getTopic()); + context.setClientHost(RemotingHelper.parseChannelRemoteAddr(ctx.channel())); + context.setSuccess(true); + context.setStatus(ConsumeConcurrentlyStatus.CONSUME_SUCCESS.toString()); + final SocketAddress storeHost = + new InetSocketAddress(brokerController.getBrokerConfig().getBrokerIP1(), brokerController + .getNettyServerConfig().getListenPort()); + + long preOffset = + this.brokerController.getConsumerOffsetManager().queryOffset( + requestHeader.getConsumerGroup(), requestHeader.getTopic(), + requestHeader.getQueueId()); + Map messageIds = + this.brokerController.getMessageStore().getMessageIds(requestHeader.getTopic(), + requestHeader.getQueueId(), preOffset, requestHeader.getCommitOffset(), storeHost); + context.setMessageIds(messageIds); + this.executeConsumeMessageHookAfter(context); + } + this.brokerController.getConsumerOffsetManager().commitOffset(requestHeader.getConsumerGroup(), + requestHeader.getTopic(), requestHeader.getQueueId(), requestHeader.getCommitOffset()); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + + private RemotingCommand queryConsumerOffset(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + final RemotingCommand response = + RemotingCommand.createResponseCommand(QueryConsumerOffsetResponseHeader.class); + final QueryConsumerOffsetResponseHeader responseHeader = + (QueryConsumerOffsetResponseHeader) response.readCustomHeader(); + final QueryConsumerOffsetRequestHeader requestHeader = + (QueryConsumerOffsetRequestHeader) request + .decodeCommandCustomHeader(QueryConsumerOffsetRequestHeader.class); + + long offset = + this.brokerController.getConsumerOffsetManager().queryOffset( + requestHeader.getConsumerGroup(), requestHeader.getTopic(), requestHeader.getQueueId()); + + // 订阅组存在 + if (offset >= 0) { + responseHeader.setOffset(offset); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + } + // 订阅组不存在 + else { + long minOffset = + this.brokerController.getMessageStore().getMinOffsetInQuque(requestHeader.getTopic(), + requestHeader.getQueueId()); + // 订阅组不存在情况下，如果这个队列的消息最小Offset是0，则表示这个Topic上线时间不长，服务器堆积的数据也不多，那么这个订阅组就从0开始消费。 + // 尤其对于Topic队列数动态扩容时，必须要从0开始消费。 + if (minOffset <= 0 + && !this.brokerController.getMessageStore().checkInDiskByConsumeOffset( + requestHeader.getTopic(), requestHeader.getQueueId(), 0)) { + responseHeader.setOffset(0L); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + } + // 新版本服务器不做消费进度纠正 + else { + response.setCode(ResponseCode.QUERY_NOT_FOUND); + response.setRemark("Not found, V3_0_6_SNAPSHOT maybe this group consumer boot first"); + } + } + + return response; + } + + + public RemotingCommand getConsumerListByGroup(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + final RemotingCommand response = + RemotingCommand.createResponseCommand(GetConsumerListByGroupResponseHeader.class); + final GetConsumerListByGroupRequestHeader requestHeader = + (GetConsumerListByGroupRequestHeader) request + .decodeCommandCustomHeader(GetConsumerListByGroupRequestHeader.class); + + ConsumerGroupInfo consumerGroupInfo = + this.brokerController.getConsumerManager().getConsumerGroupInfo( + requestHeader.getConsumerGroup()); + if (consumerGroupInfo != null) { + List clientIds = consumerGroupInfo.getAllClientId(); + if (!clientIds.isEmpty()) { + GetConsumerListByGroupResponseBody body = new GetConsumerListByGroupResponseBody(); + body.setConsumerIdList(clientIds); + response.setBody(body.encode()); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + else { + log.warn("getAllClientId failed, {} {}", requestHeader.getConsumerGroup(), + RemotingHelper.parseChannelRemoteAddr(ctx.channel())); + } + } + else { + log.warn("getConsumerGroupInfo failed, {} {}", requestHeader.getConsumerGroup(), + RemotingHelper.parseChannelRemoteAddr(ctx.channel())); + } + + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("no consumer for this group, " + requestHeader.getConsumerGroup()); + return response; + } + + + public RemotingCommand unregisterClient(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + final RemotingCommand response = + RemotingCommand.createResponseCommand(UnregisterClientResponseHeader.class); + final UnregisterClientRequestHeader requestHeader = + (UnregisterClientRequestHeader) request + .decodeCommandCustomHeader(UnregisterClientRequestHeader.class); + + ClientChannelInfo clientChannelInfo = new ClientChannelInfo(// + ctx.channel(),// + requestHeader.getClientID(),// + request.getLanguage(),// + request.getVersion()// + ); + + // 注销Producer + { + final String group = requestHeader.getProducerGroup(); + if (group != null) { + this.brokerController.getProducerManager().unregisterProducer(group, clientChannelInfo); + } + } + + // 注销Consumer + { + final String group = requestHeader.getConsumerGroup(); + if (group != null) { + this.brokerController.getConsumerManager().unregisterConsumer(group, clientChannelInfo); + } + } + + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + + public RemotingCommand heartBeat(ChannelHandlerContext ctx, RemotingCommand request) { + RemotingCommand response = RemotingCommand.createResponseCommand(null); + + HeartbeatData heartbeatData = HeartbeatData.decode(request.getBody(), HeartbeatData.class); + + ClientChannelInfo clientChannelInfo = new ClientChannelInfo(// + ctx.channel(),// + heartbeatData.getClientID(),// + request.getLanguage(),// + request.getVersion()// + ); + + // 注册Consumer + for (ConsumerData data : heartbeatData.getConsumerDataSet()) { + SubscriptionGroupConfig subscriptionGroupConfig = + this.brokerController.getSubscriptionGroupManager().findSubscriptionGroupConfig( + data.getGroupName()); + if (null != subscriptionGroupConfig) { + // 如果是单元化模式，则对 topic 进行设置 + int topicSysFlag = 0; + if (data.isUnitMode()) { + topicSysFlag = TopicSysFlag.buildSysFlag(false, true); + } + String newTopic = MixAll.getRetryTopic(data.getGroupName()); + this.brokerController.getTopicConfigManager().createTopicInSendMessageBackMethod(// + newTopic,// + subscriptionGroupConfig.getRetryQueueNums(), // + PermName.PERM_WRITE | PermName.PERM_READ, topicSysFlag); + } + + boolean changed = this.brokerController.getConsumerManager().registerConsumer(// + data.getGroupName(),// + clientChannelInfo,// + data.getConsumeType(),// + data.getMessageModel(),// + data.getConsumeFromWhere(),// + data.getSubscriptionDataSet()// + ); + + if (changed) { + log.info("registerConsumer info changed {} {}",// + data.toString(),// + RemotingHelper.parseChannelRemoteAddr(ctx.channel())// + ); + + // todo:有可能会有频繁变更 + // for (SubscriptionData subscriptionData : + // data.getSubscriptionDataSet()) { + // this.brokerController.getTopicConfigManager().updateTopicUnitSubFlag( + // subscriptionData.getTopic(), data.isUnitMode()); + // } + } + } + + // 注册Producer + for (ProducerData data : heartbeatData.getProducerDataSet()) { + this.brokerController.getProducerManager().registerProducer(data.getGroupName(), + clientChannelInfo); + } + + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/processor/EndTransactionProcessor.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/processor/EndTransactionProcessor.java new file mode 100644 index 000000000..1db52ee1b --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/processor/EndTransactionProcessor.java @@ -0,0 +1,255 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.broker.processor; + +import io.netty.channel.ChannelHandlerContext; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.rocketmq.broker.BrokerController; +import com.alibaba.rocketmq.common.TopicFilterType; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.common.message.MessageAccessor; +import com.alibaba.rocketmq.common.message.MessageConst; +import com.alibaba.rocketmq.common.message.MessageDecoder; +import com.alibaba.rocketmq.common.message.MessageExt; +import com.alibaba.rocketmq.common.protocol.ResponseCode; +import com.alibaba.rocketmq.common.protocol.header.EndTransactionRequestHeader; +import com.alibaba.rocketmq.common.sysflag.MessageSysFlag; +import com.alibaba.rocketmq.remoting.common.RemotingHelper; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; +import com.alibaba.rocketmq.remoting.netty.NettyRequestProcessor; +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; +import com.alibaba.rocketmq.store.MessageExtBrokerInner; +import com.alibaba.rocketmq.store.MessageStore; +import com.alibaba.rocketmq.store.PutMessageResult; + + +/** + * Commit或Rollback事务 + * + * @author shijia.wxr + * @since 2013-7-26 + */ +public class EndTransactionProcessor implements NettyRequestProcessor { + private static final Logger log = LoggerFactory.getLogger(LoggerName.BrokerLoggerName); + + private final BrokerController brokerController; + + + public EndTransactionProcessor(final BrokerController brokerController) { + this.brokerController = brokerController; + } + + + private MessageExtBrokerInner endMessageTransaction(MessageExt msgExt) { + MessageExtBrokerInner msgInner = new MessageExtBrokerInner(); + msgInner.setBody(msgExt.getBody()); + msgInner.setFlag(msgExt.getFlag()); + MessageAccessor.setProperties(msgInner, msgExt.getProperties()); + + TopicFilterType topicFilterType = + (msgInner.getSysFlag() & MessageSysFlag.MultiTagsFlag) == MessageSysFlag.MultiTagsFlag ? TopicFilterType.MULTI_TAG + : TopicFilterType.SINGLE_TAG; + long tagsCodeValue = MessageExtBrokerInner.tagsString2tagsCode(topicFilterType, msgInner.getTags()); + msgInner.setTagsCode(tagsCodeValue); + msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgExt.getProperties())); + + msgInner.setSysFlag(msgExt.getSysFlag()); + msgInner.setBornTimestamp(msgExt.getBornTimestamp()); + msgInner.setBornHost(msgExt.getBornHost()); + msgInner.setStoreHost(msgExt.getStoreHost()); + msgInner.setReconsumeTimes(msgExt.getReconsumeTimes()); + + msgInner.setWaitStoreMsgOK(false); + MessageAccessor.clearProperty(msgInner, MessageConst.PROPERTY_DELAY_TIME_LEVEL); + + msgInner.setTopic(msgExt.getTopic()); + msgInner.setQueueId(msgExt.getQueueId()); + + return msgInner; + } + + private static final Logger logTransaction = LoggerFactory.getLogger(LoggerName.TransactionLoggerName); + + + @Override + public RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + final EndTransactionRequestHeader requestHeader = + (EndTransactionRequestHeader) request + .decodeCommandCustomHeader(EndTransactionRequestHeader.class); + + // 回查应答 + if (requestHeader.getFromTransactionCheck()) { + switch (requestHeader.getCommitOrRollback()) { + // 不提交也不回滚 + case MessageSysFlag.TransactionNotType: { + logTransaction.warn("check producer[{}] transaction state, but it's pending status.\n"// + + "RequestHeader: {} Remark: {}",// + RemotingHelper.parseChannelRemoteAddr(ctx.channel()), // + requestHeader.toString(),// + request.getRemark()); + return null; + } + // 提交 + case MessageSysFlag.TransactionCommitType: { + logTransaction.warn( + "check producer[{}] transaction state, the producer commit the message.\n"// + + "RequestHeader: {} Remark: {}",// + RemotingHelper.parseChannelRemoteAddr(ctx.channel()), // + requestHeader.toString(),// + request.getRemark()); + + break; + } + // 回滚 + case MessageSysFlag.TransactionRollbackType: { + logTransaction.warn( + "check producer[{}] transaction state, the producer rollback the message.\n"// + + "RequestHeader: {} Remark: {}",// + RemotingHelper.parseChannelRemoteAddr(ctx.channel()), // + requestHeader.toString(),// + request.getRemark()); + break; + } + default: + return null; + } + } + // 正常提交回滚 + else { + switch (requestHeader.getCommitOrRollback()) { + // 不提交也不回滚 + case MessageSysFlag.TransactionNotType: { + logTransaction.warn( + "the producer[{}] end transaction in sending message, and it's pending status.\n"// + + "RequestHeader: {} Remark: {}",// + RemotingHelper.parseChannelRemoteAddr(ctx.channel()), // + requestHeader.toString(),// + request.getRemark()); + return null; + } + // 提交 + case MessageSysFlag.TransactionCommitType: { + break; + } + // 回滚 + case MessageSysFlag.TransactionRollbackType: { + logTransaction.warn( + "the producer[{}] end transaction in sending message, rollback the message.\n"// + + "RequestHeader: {} Remark: {}",// + RemotingHelper.parseChannelRemoteAddr(ctx.channel()), // + requestHeader.toString(),// + request.getRemark()); + break; + } + default: + return null; + } + } + + final MessageExt msgExt = + this.brokerController.getMessageStore().lookMessageByOffset( + requestHeader.getCommitLogOffset()); + if (msgExt != null) { + // 校验Producer Group + final String pgroupRead = msgExt.getProperty(MessageConst.PROPERTY_PRODUCER_GROUP); + if (!pgroupRead.equals(requestHeader.getProducerGroup())) { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("the producer group wrong"); + return response; + } + + // 校验Transaction State Table Offset + if (msgExt.getQueueOffset() != requestHeader.getTranStateTableOffset()) { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("the transaction state table offset wrong"); + return response; + } + + // 校验Commit Log Offset + if (msgExt.getCommitLogOffset() != requestHeader.getCommitLogOffset()) { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("the commit log offset wrong"); + return response; + } + + MessageExtBrokerInner msgInner = this.endMessageTransaction(msgExt); + msgInner.setSysFlag(MessageSysFlag.resetTransactionValue(msgInner.getSysFlag(), + requestHeader.getCommitOrRollback())); + + msgInner.setQueueOffset(requestHeader.getTranStateTableOffset()); + msgInner.setPreparedTransactionOffset(requestHeader.getCommitLogOffset()); + msgInner.setStoreTimestamp(msgExt.getStoreTimestamp()); + if (MessageSysFlag.TransactionRollbackType == requestHeader.getCommitOrRollback()) { + msgInner.setBody(null); + } + + final MessageStore messageStore = this.brokerController.getMessageStore(); + final PutMessageResult putMessageResult = messageStore.putMessage(msgInner); + if (putMessageResult != null) { + switch (putMessageResult.getPutMessageStatus()) { + // Success + case PUT_OK: + case FLUSH_DISK_TIMEOUT: + case FLUSH_SLAVE_TIMEOUT: + case SLAVE_NOT_AVAILABLE: + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + break; + + // Failed + case CREATE_MAPEDFILE_FAILED: + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("create maped file failed."); + break; + case MESSAGE_ILLEGAL: + response.setCode(ResponseCode.MESSAGE_ILLEGAL); + response.setRemark("the message is illegal, maybe length not matched."); + break; + case SERVICE_NOT_AVAILABLE: + response.setCode(ResponseCode.SERVICE_NOT_AVAILABLE); + response.setRemark("service not available now."); + break; + case UNKNOWN_ERROR: + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("UNKNOWN_ERROR"); + break; + default: + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("UNKNOWN_ERROR DEFAULT"); + break; + } + + return response; + } + else { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("store putMessage return null"); + } + } + else { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("find prepared transaction message failed"); + return response; + } + + return response; + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/processor/ForwardRequestProcessor.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/processor/ForwardRequestProcessor.java new file mode 100644 index 000000000..226ede0ec --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/processor/ForwardRequestProcessor.java @@ -0,0 +1,51 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.broker.processor; + +import io.netty.channel.ChannelHandlerContext; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.rocketmq.broker.BrokerController; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.remoting.netty.NettyRequestProcessor; +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; + + +/** + * 向Client转发请求，通常用于管理、监控、统计目的 + * + * @author shijia.wxr + * @since 2013-7-26 + */ +public class ForwardRequestProcessor implements NettyRequestProcessor { + private static final Logger log = LoggerFactory.getLogger(LoggerName.BrokerLoggerName); + + private final BrokerController brokerController; + + + public ForwardRequestProcessor(final BrokerController brokerController) { + this.brokerController = brokerController; + } + + + @Override + public RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand request) { + // TODO Auto-generated method stub + return null; + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/processor/PullMessageProcessor.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/processor/PullMessageProcessor.java new file mode 100644 index 000000000..2df20c90c --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/processor/PullMessageProcessor.java @@ -0,0 +1,515 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.broker.processor; + +import io.netty.channel.Channel; +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelFutureListener; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.FileRegion; + +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.util.List; +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.rocketmq.broker.BrokerController; +import com.alibaba.rocketmq.broker.client.ConsumerGroupInfo; +import com.alibaba.rocketmq.broker.longpolling.PullRequest; +import com.alibaba.rocketmq.broker.mqtrace.ConsumeMessageContext; +import com.alibaba.rocketmq.broker.mqtrace.ConsumeMessageHook; +import com.alibaba.rocketmq.broker.pagecache.ManyMessageTransfer; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.TopicConfig; +import com.alibaba.rocketmq.common.TopicFilterType; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.common.constant.PermName; +import com.alibaba.rocketmq.common.filter.FilterAPI; +import com.alibaba.rocketmq.common.help.FAQUrl; +import com.alibaba.rocketmq.common.message.MessageDecoder; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.common.protocol.ResponseCode; +import com.alibaba.rocketmq.common.protocol.header.PullMessageRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.PullMessageResponseHeader; +import com.alibaba.rocketmq.common.protocol.heartbeat.MessageModel; +import com.alibaba.rocketmq.common.protocol.heartbeat.SubscriptionData; +import com.alibaba.rocketmq.common.protocol.topic.OffsetMovedEvent; +import com.alibaba.rocketmq.common.subscription.SubscriptionGroupConfig; +import com.alibaba.rocketmq.common.sysflag.PullSysFlag; +import com.alibaba.rocketmq.remoting.common.RemotingHelper; +import com.alibaba.rocketmq.remoting.common.RemotingUtil; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; +import com.alibaba.rocketmq.remoting.netty.NettyRequestProcessor; +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; +import com.alibaba.rocketmq.store.GetMessageResult; +import com.alibaba.rocketmq.store.MessageExtBrokerInner; +import com.alibaba.rocketmq.store.PutMessageResult; +import com.alibaba.rocketmq.store.config.BrokerRole; + + +/** + * 拉消息请求处理 + * + * @author shijia.wxr + * @since 2013-7-26 + */ +public class PullMessageProcessor implements NettyRequestProcessor { + private static final Logger log = LoggerFactory.getLogger(LoggerName.BrokerLoggerName); + + private final BrokerController brokerController; + + + public PullMessageProcessor(final BrokerController brokerController) { + this.brokerController = brokerController; + } + + + @Override + public RemotingCommand processRequest(final ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + return this.processRequest(ctx.channel(), request, true); + } + + + public void excuteRequestWhenWakeup(final Channel channel, final RemotingCommand request) + throws RemotingCommandException { + Runnable run = new Runnable() { + @Override + public void run() { + try { + final RemotingCommand response = + PullMessageProcessor.this.processRequest(channel, request, false); + + if (response != null) { + response.setOpaque(request.getOpaque()); + response.markResponseType(); + try { + channel.writeAndFlush(response).addListener(new ChannelFutureListener() { + @Override + public void operationComplete(ChannelFuture future) throws Exception { + if (!future.isSuccess()) { + log.error("processRequestWrapper response to " + + future.channel().remoteAddress() + " failed", + future.cause()); + log.error(request.toString()); + log.error(response.toString()); + } + } + }); + } + catch (Throwable e) { + log.error("processRequestWrapper process request over, but response failed", e); + log.error(request.toString()); + log.error(response.toString()); + } + } + } + catch (RemotingCommandException e1) { + log.error("excuteRequestWhenWakeup run", e1); + } + } + }; + + this.brokerController.getPullMessageExecutor().submit(run); + } + + + private void generateOffsetMovedEvent(final OffsetMovedEvent event) { + try { + MessageExtBrokerInner msgInner = new MessageExtBrokerInner(); + msgInner.setTopic(MixAll.OFFSET_MOVED_EVENT); + msgInner.setTags(event.getConsumerGroup()); + msgInner.setDelayTimeLevel(0); + msgInner.setKeys(event.getConsumerGroup()); + msgInner.setBody(event.encode()); + msgInner.setFlag(0); + msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties())); + msgInner.setTagsCode(MessageExtBrokerInner.tagsString2tagsCode(TopicFilterType.SINGLE_TAG, + msgInner.getTags())); + + msgInner.setQueueId(0); + msgInner.setSysFlag(0); + msgInner.setBornTimestamp(System.currentTimeMillis()); + msgInner.setBornHost(RemotingUtil.string2SocketAddress(this.brokerController.getBrokerAddr())); + msgInner.setStoreHost(msgInner.getBornHost()); + + msgInner.setReconsumeTimes(0); + + PutMessageResult putMessageResult = this.brokerController.getMessageStore().putMessage(msgInner); + } + catch (Exception e) { + log.warn(String.format("generateOffsetMovedEvent Exception, %s", event.toString()), e); + } + } + + + private RemotingCommand processRequest(final Channel channel, RemotingCommand request, + boolean brokerAllowSuspend) throws RemotingCommandException { + RemotingCommand response = RemotingCommand.createResponseCommand(PullMessageResponseHeader.class); + final PullMessageResponseHeader responseHeader = + (PullMessageResponseHeader) response.readCustomHeader(); + final PullMessageRequestHeader requestHeader = + (PullMessageRequestHeader) request.decodeCommandCustomHeader(PullMessageRequestHeader.class); + + // 由于使用sendfile，所以必须要设置 + response.setOpaque(request.getOpaque()); + + if (log.isDebugEnabled()) { + log.debug("receive PullMessage request command, " + request); + } + + // 检查Broker权限 + if (!PermName.isReadable(this.brokerController.getBrokerConfig().getBrokerPermission())) { + response.setCode(ResponseCode.NO_PERMISSION); + response.setRemark("the broker[" + this.brokerController.getBrokerConfig().getBrokerIP1() + + "] pulling message is forbidden"); + return response; + } + + // 确保订阅组存在 + SubscriptionGroupConfig subscriptionGroupConfig = + this.brokerController.getSubscriptionGroupManager().findSubscriptionGroupConfig( + requestHeader.getConsumerGroup()); + if (null == subscriptionGroupConfig) { + response.setCode(ResponseCode.SUBSCRIPTION_GROUP_NOT_EXIST); + response.setRemark("subscription group not exist, " + requestHeader.getConsumerGroup() + " " + + FAQUrl.suggestTodo(FAQUrl.SUBSCRIPTION_GROUP_NOT_EXIST)); + return response; + } + + // 这个订阅组是否可以消费消息 + if (!subscriptionGroupConfig.isConsumeEnable()) { + response.setCode(ResponseCode.NO_PERMISSION); + response.setRemark("subscription group no permission, " + requestHeader.getConsumerGroup()); + return response; + } + + final boolean hasSuspendFlag = PullSysFlag.hasSuspendFlag(requestHeader.getSysFlag()); + final boolean hasCommitOffsetFlag = PullSysFlag.hasCommitOffsetFlag(requestHeader.getSysFlag()); + final boolean hasSubscriptionFlag = PullSysFlag.hasSubscriptionFlag(requestHeader.getSysFlag()); + + final long suspendTimeoutMillisLong = hasSuspendFlag ? requestHeader.getSuspendTimeoutMillis() : 0; + + // 检查topic是否存在 + TopicConfig topicConfig = + this.brokerController.getTopicConfigManager().selectTopicConfig(requestHeader.getTopic()); + if (null == topicConfig) { + log.error("the topic " + requestHeader.getTopic() + " not exist, consumer: " + + RemotingHelper.parseChannelRemoteAddr(channel)); + response.setCode(ResponseCode.TOPIC_NOT_EXIST); + response.setRemark("topic[" + requestHeader.getTopic() + "] not exist, apply first please!" + + FAQUrl.suggestTodo(FAQUrl.APPLY_TOPIC_URL)); + return response; + } + + // 检查topic权限 + if (!PermName.isReadable(topicConfig.getPerm())) { + response.setCode(ResponseCode.NO_PERMISSION); + response.setRemark("the topic[" + requestHeader.getTopic() + "] pulling message is forbidden"); + return response; + } + + // 检查队列有效性 + if (requestHeader.getQueueId() < 0 || requestHeader.getQueueId() >= topicConfig.getReadQueueNums()) { + String errorInfo = + "queueId[" + requestHeader.getQueueId() + "] is illagal,Topic :" + + requestHeader.getTopic() + " topicConfig.readQueueNums: " + + topicConfig.getReadQueueNums() + " consumer: " + channel.remoteAddress(); + log.warn(errorInfo); + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark(errorInfo); + return response; + } + + // 订阅关系处理 + SubscriptionData subscriptionData = null; + if (hasSubscriptionFlag) { + try { + subscriptionData = + FilterAPI.buildSubscriptionData(requestHeader.getConsumerGroup(), + requestHeader.getTopic(), requestHeader.getSubscription()); + } + catch (Exception e) { + log.warn("parse the consumer's subscription[{}] failed, group: {}", + requestHeader.getSubscription(),// + requestHeader.getConsumerGroup()); + response.setCode(ResponseCode.SUBSCRIPTION_PARSE_FAILED); + response.setRemark("parse the consumer's subscription failed"); + return response; + } + } + else { + ConsumerGroupInfo consumerGroupInfo = + this.brokerController.getConsumerManager().getConsumerGroupInfo( + requestHeader.getConsumerGroup()); + if (null == consumerGroupInfo) { + log.warn("the consumer's group info not exist, group: {}", requestHeader.getConsumerGroup()); + response.setCode(ResponseCode.SUBSCRIPTION_NOT_EXIST); + response.setRemark("the consumer's group info not exist" + + FAQUrl.suggestTodo(FAQUrl.SAME_GROUP_DIFFERENT_TOPIC)); + return response; + } + + if (!subscriptionGroupConfig.isConsumeBroadcastEnable() // + && consumerGroupInfo.getMessageModel() == MessageModel.BROADCASTING) { + response.setCode(ResponseCode.NO_PERMISSION); + response.setRemark("the consumer group[" + requestHeader.getConsumerGroup() + + "] can not consume by broadcast way"); + return response; + } + + subscriptionData = consumerGroupInfo.findSubscriptionData(requestHeader.getTopic()); + if (null == subscriptionData) { + log.warn("the consumer's subscription not exist, group: {}", requestHeader.getConsumerGroup()); + response.setCode(ResponseCode.SUBSCRIPTION_NOT_EXIST); + response.setRemark("the consumer's subscription not exist" + + FAQUrl.suggestTodo(FAQUrl.SAME_GROUP_DIFFERENT_TOPIC)); + return response; + } + + // 判断Broker的订阅关系版本是否最新 + if (subscriptionData.getSubVersion() < requestHeader.getSubVersion()) { + log.warn("the broker's subscription is not latest, group: {} {}", + requestHeader.getConsumerGroup(), subscriptionData.getSubString()); + response.setCode(ResponseCode.SUBSCRIPTION_NOT_LATEST); + response.setRemark("the consumer's subscription not latest"); + return response; + } + } + + final GetMessageResult getMessageResult = + this.brokerController.getMessageStore().getMessage(requestHeader.getConsumerGroup(), + requestHeader.getTopic(), requestHeader.getQueueId(), requestHeader.getQueueOffset(), + requestHeader.getMaxMsgNums(), subscriptionData); + if (getMessageResult != null) { + response.setRemark(getMessageResult.getStatus().name()); + responseHeader.setNextBeginOffset(getMessageResult.getNextBeginOffset()); + responseHeader.setMinOffset(getMessageResult.getMinOffset()); + responseHeader.setMaxOffset(getMessageResult.getMaxOffset()); + + // 消费较慢，重定向到另外一台机器 + if (getMessageResult.isSuggestPullingFromSlave()) { + responseHeader.setSuggestWhichBrokerId(subscriptionGroupConfig + .getWhichBrokerWhenConsumeSlowly()); + } + // 消费正常，按照订阅组配置重定向 + else { + responseHeader.setSuggestWhichBrokerId(subscriptionGroupConfig.getBrokerId()); + } + + switch (getMessageResult.getStatus()) { + case FOUND: + response.setCode(ResponseCode.SUCCESS); + + // 消息轨迹：记录客户端拉取的消息记录（不表示消费成功） + if (this.hasConsumeMessageHook()) { + // 执行hook + ConsumeMessageContext context = new ConsumeMessageContext(); + context.setConsumerGroup(requestHeader.getConsumerGroup()); + context.setTopic(requestHeader.getTopic()); + context.setClientHost(RemotingHelper.parseChannelRemoteAddr(channel)); + context.setStoreHost(this.brokerController.getBrokerAddr()); + context.setQueueId(requestHeader.getQueueId()); + + final SocketAddress storeHost = + new InetSocketAddress(brokerController.getBrokerConfig().getBrokerIP1(), + brokerController.getNettyServerConfig().getListenPort()); + Map messageIds = + this.brokerController.getMessageStore().getMessageIds(requestHeader.getTopic(), + requestHeader.getQueueId(), requestHeader.getQueueOffset(), + requestHeader.getQueueOffset() + getMessageResult.getMessageCount(), + storeHost); + context.setMessageIds(messageIds); + context.setBodyLength(getMessageResult.getBufferTotalSize() + / getMessageResult.getMessageCount()); + this.executeConsumeMessageHookBefore(context); + } + + break; + case MESSAGE_WAS_REMOVING: + response.setCode(ResponseCode.PULL_RETRY_IMMEDIATELY); + break; + // 这两个返回值都表示服务器暂时没有这个队列，应该立刻将客户端Offset重置为0 + case NO_MATCHED_LOGIC_QUEUE: + case NO_MESSAGE_IN_QUEUE: + if (0 != requestHeader.getQueueOffset()) { + response.setCode(ResponseCode.PULL_OFFSET_MOVED); + + // XXX: warn and notify me + log.info( + "the broker store no queue data, fix the request offset {} to {}, Topic: {} QueueId: {} Consumer Group: {}",// + requestHeader.getQueueOffset(), // + getMessageResult.getNextBeginOffset(), // + requestHeader.getTopic(),// + requestHeader.getQueueId(),// + requestHeader.getConsumerGroup()// + ); + } + else { + response.setCode(ResponseCode.PULL_NOT_FOUND); + } + break; + case NO_MATCHED_MESSAGE: + response.setCode(ResponseCode.PULL_RETRY_IMMEDIATELY); + break; + case OFFSET_FOUND_NULL: + response.setCode(ResponseCode.PULL_NOT_FOUND); + break; + case OFFSET_OVERFLOW_BADLY: + response.setCode(ResponseCode.PULL_OFFSET_MOVED); + // XXX: warn and notify me + log.info("the request offset: " + requestHeader.getQueueOffset() + + " over flow badly, broker max offset: " + getMessageResult.getMaxOffset() + + ", consumer: " + channel.remoteAddress()); + break; + case OFFSET_OVERFLOW_ONE: + response.setCode(ResponseCode.PULL_NOT_FOUND); + break; + case OFFSET_TOO_SMALL: + response.setCode(ResponseCode.PULL_OFFSET_MOVED); + // XXX: warn and notify me + log.info("the request offset: " + requestHeader.getQueueOffset() + + " too small, broker min offset: " + getMessageResult.getMinOffset() + + ", consumer: " + channel.remoteAddress()); + break; + default: + assert false; + break; + } + + switch (response.getCode()) { + case ResponseCode.SUCCESS: + // 统计 + this.brokerController.getBrokerStatsManager().incGroupGetNums( + requestHeader.getConsumerGroup(), requestHeader.getTopic(), + getMessageResult.getMessageCount()); + + this.brokerController.getBrokerStatsManager().incGroupGetSize( + requestHeader.getConsumerGroup(), requestHeader.getTopic(), + getMessageResult.getBufferTotalSize()); + + this.brokerController.getBrokerStatsManager().incBrokerGetNums( + getMessageResult.getMessageCount()); + + try { + FileRegion fileRegion = + new ManyMessageTransfer(response.encodeHeader(getMessageResult + .getBufferTotalSize()), getMessageResult); + channel.writeAndFlush(fileRegion).addListener(new ChannelFutureListener() { + @Override + public void operationComplete(ChannelFuture future) throws Exception { + getMessageResult.release(); + if (!future.isSuccess()) { + log.error( + "transfer many message by pagecache failed, " + channel.remoteAddress(), + future.cause()); + } + } + }); + } + catch (Throwable e) { + log.error("", e); + getMessageResult.release(); + } + + response = null; + break; + case ResponseCode.PULL_NOT_FOUND: + // 长轮询 + if (brokerAllowSuspend && hasSuspendFlag) { + long pollingTimeMills = suspendTimeoutMillisLong; + if (this.brokerController.getBrokerConfig().isLongPollingEnable()) { + pollingTimeMills = this.brokerController.getBrokerConfig().getShortPollingTimeMills(); + } + + PullRequest pullRequest = + new PullRequest(request, channel, pollingTimeMills, this.brokerController + .getMessageStore().now(), requestHeader.getQueueOffset()); + this.brokerController.getPullRequestHoldService().suspendPullRequest( + requestHeader.getTopic(), requestHeader.getQueueId(), pullRequest); + response = null; + break; + } + + // 向Consumer返回应答 + case ResponseCode.PULL_RETRY_IMMEDIATELY: + break; + case ResponseCode.PULL_OFFSET_MOVED: + MessageQueue mq = new MessageQueue(); + mq.setTopic(requestHeader.getTopic()); + mq.setQueueId(requestHeader.getQueueId()); + mq.setBrokerName(this.brokerController.getBrokerConfig().getBrokerName()); + + OffsetMovedEvent event = new OffsetMovedEvent(); + event.setConsumerGroup(requestHeader.getConsumerGroup()); + event.setMessageQueue(mq); + event.setOffsetRequest(requestHeader.getQueueOffset()); + event.setOffsetNew(getMessageResult.getNextBeginOffset()); + this.generateOffsetMovedEvent(event); + break; + default: + assert false; + } + } + else { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("store getMessage return null"); + } + + // 存储Consumer消费进度 + boolean storeOffsetEnable = brokerAllowSuspend; // 说明是首次调用，相对于长轮询通知 + storeOffsetEnable = storeOffsetEnable && hasCommitOffsetFlag; // 说明Consumer设置了标志位 + storeOffsetEnable = storeOffsetEnable // 只有Master支持存储offset + && this.brokerController.getMessageStoreConfig().getBrokerRole() != BrokerRole.SLAVE; + if (storeOffsetEnable) { + this.brokerController.getConsumerOffsetManager().commitOffset(requestHeader.getConsumerGroup(), + requestHeader.getTopic(), requestHeader.getQueueId(), requestHeader.getCommitOffset()); + } + + return response; + } + + /** + * 发送每条消息会回调 + */ + private List consumeMessageHookList; + + + public boolean hasConsumeMessageHook() { + return consumeMessageHookList != null && !this.consumeMessageHookList.isEmpty(); + } + + + public void registerConsumeMessageHook(List sendMessageHookList) { + this.consumeMessageHookList = sendMessageHookList; + } + + + public void executeConsumeMessageHookBefore(final ConsumeMessageContext context) { + if (hasConsumeMessageHook()) { + for (ConsumeMessageHook hook : this.consumeMessageHookList) { + try { + hook.consumeMessageBefore(context); + } + catch (Throwable e) { + } + } + } + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/processor/QueryMessageProcessor.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/processor/QueryMessageProcessor.java new file mode 100644 index 000000000..61ed9f49a --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/processor/QueryMessageProcessor.java @@ -0,0 +1,173 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.broker.processor; + +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelFutureListener; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.FileRegion; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.rocketmq.broker.BrokerController; +import com.alibaba.rocketmq.broker.pagecache.OneMessageTransfer; +import com.alibaba.rocketmq.broker.pagecache.QueryMessageTransfer; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.common.protocol.RequestCode; +import com.alibaba.rocketmq.common.protocol.ResponseCode; +import com.alibaba.rocketmq.common.protocol.header.QueryMessageRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.QueryMessageResponseHeader; +import com.alibaba.rocketmq.common.protocol.header.ViewMessageRequestHeader; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; +import com.alibaba.rocketmq.remoting.netty.NettyRequestProcessor; +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; +import com.alibaba.rocketmq.store.QueryMessageResult; +import com.alibaba.rocketmq.store.SelectMapedBufferResult; + + +/** + * 查询消息请求处理 + * + * @author shijia.wxr + * @since 2013-7-26 + */ +public class QueryMessageProcessor implements NettyRequestProcessor { + private static final Logger log = LoggerFactory.getLogger(LoggerName.BrokerLoggerName); + + private final BrokerController brokerController; + + + public QueryMessageProcessor(final BrokerController brokerController) { + this.brokerController = brokerController; + } + + + @Override + public RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + switch (request.getCode()) { + case RequestCode.QUERY_MESSAGE: + return this.queryMessage(ctx, request); + case RequestCode.VIEW_MESSAGE_BY_ID: + return this.viewMessageById(ctx, request); + default: + break; + } + + return null; + } + + + public RemotingCommand queryMessage(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + final RemotingCommand response = + RemotingCommand.createResponseCommand(QueryMessageResponseHeader.class); + final QueryMessageResponseHeader responseHeader = + (QueryMessageResponseHeader) response.readCustomHeader(); + final QueryMessageRequestHeader requestHeader = + (QueryMessageRequestHeader) request + .decodeCommandCustomHeader(QueryMessageRequestHeader.class); + + // 由于使用sendfile，所以必须要设置 + response.setOpaque(request.getOpaque()); + + final QueryMessageResult queryMessageResult = + this.brokerController.getMessageStore().queryMessage(requestHeader.getTopic(), + requestHeader.getKey(), requestHeader.getMaxNum(), requestHeader.getBeginTimestamp(), + requestHeader.getEndTimestamp()); + assert queryMessageResult != null; + + responseHeader.setIndexLastUpdatePhyoffset(queryMessageResult.getIndexLastUpdatePhyoffset()); + responseHeader.setIndexLastUpdateTimestamp(queryMessageResult.getIndexLastUpdateTimestamp()); + + // 说明找到消息 + if (queryMessageResult.getBufferTotalSize() > 0) { + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + + try { + FileRegion fileRegion = + new QueryMessageTransfer(response.encodeHeader(queryMessageResult + .getBufferTotalSize()), queryMessageResult); + ctx.channel().writeAndFlush(fileRegion).addListener(new ChannelFutureListener() { + @Override + public void operationComplete(ChannelFuture future) throws Exception { + queryMessageResult.release(); + if (!future.isSuccess()) { + log.error("transfer query message by pagecache failed, ", future.cause()); + } + } + }); + } + catch (Throwable e) { + log.error("", e); + queryMessageResult.release(); + } + + return null; + } + + response.setCode(ResponseCode.QUERY_NOT_FOUND); + response.setRemark("can not find message, maybe time range not correct"); + return response; + } + + + public RemotingCommand viewMessageById(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + final ViewMessageRequestHeader requestHeader = + (ViewMessageRequestHeader) request.decodeCommandCustomHeader(ViewMessageRequestHeader.class); + + // 由于使用sendfile，所以必须要设置 + response.setOpaque(request.getOpaque()); + + final SelectMapedBufferResult selectMapedBufferResult = + this.brokerController.getMessageStore().selectOneMessageByOffset(requestHeader.getOffset()); + if (selectMapedBufferResult != null) { + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + + try { + FileRegion fileRegion = + new OneMessageTransfer(response.encodeHeader(selectMapedBufferResult.getSize()), + selectMapedBufferResult); + ctx.channel().writeAndFlush(fileRegion).addListener(new ChannelFutureListener() { + @Override + public void operationComplete(ChannelFuture future) throws Exception { + selectMapedBufferResult.release(); + if (!future.isSuccess()) { + log.error("transfer one message by pagecache failed, ", future.cause()); + } + } + }); + } + catch (Throwable e) { + log.error("", e); + selectMapedBufferResult.release(); + } + + return null; + } + else { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("can not find message by the offset, " + requestHeader.getOffset()); + } + + return response; + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/processor/SendMessageProcessor.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/processor/SendMessageProcessor.java new file mode 100644 index 000000000..d493adbc0 --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/processor/SendMessageProcessor.java @@ -0,0 +1,669 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.broker.processor; + +import io.netty.channel.ChannelHandlerContext; + +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.rocketmq.broker.BrokerController; +import com.alibaba.rocketmq.broker.mqtrace.ConsumeMessageContext; +import com.alibaba.rocketmq.broker.mqtrace.ConsumeMessageHook; +import com.alibaba.rocketmq.broker.mqtrace.SendMessageContext; +import com.alibaba.rocketmq.broker.mqtrace.SendMessageHook; +import com.alibaba.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.TopicConfig; +import com.alibaba.rocketmq.common.TopicFilterType; +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.common.constant.PermName; +import com.alibaba.rocketmq.common.help.FAQUrl; +import com.alibaba.rocketmq.common.message.MessageAccessor; +import com.alibaba.rocketmq.common.message.MessageConst; +import com.alibaba.rocketmq.common.message.MessageDecoder; +import com.alibaba.rocketmq.common.message.MessageExt; +import com.alibaba.rocketmq.common.protocol.RequestCode; +import com.alibaba.rocketmq.common.protocol.ResponseCode; +import com.alibaba.rocketmq.common.protocol.header.ConsumerSendMsgBackRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.SendMessageRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.SendMessageRequestHeaderV2; +import com.alibaba.rocketmq.common.protocol.header.SendMessageResponseHeader; +import com.alibaba.rocketmq.common.subscription.SubscriptionGroupConfig; +import com.alibaba.rocketmq.common.sysflag.MessageSysFlag; +import com.alibaba.rocketmq.common.sysflag.TopicSysFlag; +import com.alibaba.rocketmq.remoting.common.RemotingHelper; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; +import com.alibaba.rocketmq.remoting.netty.NettyRequestProcessor; +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; +import com.alibaba.rocketmq.store.MessageExtBrokerInner; +import com.alibaba.rocketmq.store.PutMessageResult; +import com.alibaba.rocketmq.store.config.StorePathConfigHelper; + + +/** + * 处理客户端发送消息的请求 + * + * @author shijia.wxr + * @since 2013-7-26 + */ +public class SendMessageProcessor implements NettyRequestProcessor { + private static final Logger log = LoggerFactory.getLogger(LoggerName.BrokerLoggerName); + + private final static int DLQ_NUMS_PER_GROUP = 1; + private final BrokerController brokerController; + private final Random random = new Random(System.currentTimeMillis()); + private final SocketAddress storeHost; + + + public SendMessageProcessor(final BrokerController brokerController) { + this.brokerController = brokerController; + this.storeHost = + new InetSocketAddress(brokerController.getBrokerConfig().getBrokerIP1(), brokerController + .getNettyServerConfig().getListenPort()); + } + + + @Override + public RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + SendMessageRequestHeaderV2 requestHeaderV2 = null; + + switch (request.getCode()) { + case RequestCode.SEND_MESSAGE_V2: + requestHeaderV2 = + (SendMessageRequestHeaderV2) request + .decodeCommandCustomHeader(SendMessageRequestHeaderV2.class); + case RequestCode.SEND_MESSAGE: + SendMessageContext mqtraceContext = null; + SendMessageRequestHeader requestHeader = null; + + if (null == requestHeaderV2) { + requestHeader = + (SendMessageRequestHeader) request + .decodeCommandCustomHeader(SendMessageRequestHeader.class); + } + else { + requestHeader = SendMessageRequestHeaderV2.createSendMessageRequestHeaderV1(requestHeaderV2); + } + + // 消息轨迹：记录到达 broker 的消息 + if (this.hasSendMessageHook()) { + mqtraceContext = new SendMessageContext(); + mqtraceContext.setProducerGroup(requestHeader.getProducerGroup()); + mqtraceContext.setTopic(requestHeader.getTopic()); + mqtraceContext.setMsgProps(requestHeader.getProperties()); + mqtraceContext.setBornHost(RemotingHelper.parseChannelRemoteAddr(ctx.channel())); + mqtraceContext.setBrokerAddr(this.brokerController.getBrokerAddr()); + this.executeSendMessageHookBefore(ctx, request, mqtraceContext); + } + + final RemotingCommand response = this.sendMessage(ctx, request, mqtraceContext, requestHeader); + + // 消息轨迹：记录发送成功的消息 + if (this.hasSendMessageHook()) { + this.executeSendMessageHookAfter(response, mqtraceContext); + } + return response; + case RequestCode.CONSUMER_SEND_MSG_BACK: + return this.consumerSendMsgBack(ctx, request); + default: + break; + } + return null; + } + + + private RemotingCommand consumerSendMsgBack(final ChannelHandlerContext ctx, final RemotingCommand request) + throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + final ConsumerSendMsgBackRequestHeader requestHeader = + (ConsumerSendMsgBackRequestHeader) request + .decodeCommandCustomHeader(ConsumerSendMsgBackRequestHeader.class); + + // 消息轨迹：记录消费失败的消息 + if (this.hasConsumeMessageHook() && !UtilAll.isBlank(requestHeader.getOriginMsgId())) { + // 执行hook + ConsumeMessageContext context = new ConsumeMessageContext(); + context.setConsumerGroup(requestHeader.getGroup()); + context.setTopic(requestHeader.getOriginTopic()); + context.setClientHost(RemotingHelper.parseChannelRemoteAddr(ctx.channel())); + context.setSuccess(false); + context.setStatus(ConsumeConcurrentlyStatus.RECONSUME_LATER.toString()); + + Map messageIds = new HashMap(); + messageIds.put(requestHeader.getOriginMsgId(), requestHeader.getOffset()); + context.setMessageIds(messageIds); + this.executeConsumeMessageHookAfter(context); + } + + // 确保订阅组存在 + SubscriptionGroupConfig subscriptionGroupConfig = + this.brokerController.getSubscriptionGroupManager().findSubscriptionGroupConfig( + requestHeader.getGroup()); + if (null == subscriptionGroupConfig) { + response.setCode(ResponseCode.SUBSCRIPTION_GROUP_NOT_EXIST); + response.setRemark("subscription group not exist, " + requestHeader.getGroup() + " " + + FAQUrl.suggestTodo(FAQUrl.SUBSCRIPTION_GROUP_NOT_EXIST)); + return response; + } + + // 检查Broker权限 + if (!PermName.isWriteable(this.brokerController.getBrokerConfig().getBrokerPermission())) { + response.setCode(ResponseCode.NO_PERMISSION); + response.setRemark("the broker[" + this.brokerController.getBrokerConfig().getBrokerIP1() + + "] sending message is forbidden"); + return response; + } + + // 如果重试队列数目为0，则直接丢弃消息 + if (subscriptionGroupConfig.getRetryQueueNums() <= 0) { + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + String newTopic = MixAll.getRetryTopic(requestHeader.getGroup()); + int queueIdInt = + Math.abs(this.random.nextInt() % 99999999) % subscriptionGroupConfig.getRetryQueueNums(); + + // 如果是单元化模式，则对 topic 进行设置 + int topicSysFlag = 0; + if (requestHeader.isUnitMode()) { + topicSysFlag = TopicSysFlag.buildSysFlag(false, true); + } + + // 检查topic是否存在 + TopicConfig topicConfig = + this.brokerController.getTopicConfigManager().createTopicInSendMessageBackMethod(// + newTopic,// + subscriptionGroupConfig.getRetryQueueNums(), // + PermName.PERM_WRITE | PermName.PERM_READ, topicSysFlag); + if (null == topicConfig) { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("topic[" + newTopic + "] not exist"); + return response; + } + + // 检查topic权限 + if (!PermName.isWriteable(topicConfig.getPerm())) { + response.setCode(ResponseCode.NO_PERMISSION); + response.setRemark(String.format("the topic[%s] sending message is forbidden", newTopic)); + return response; + } + + // 查询消息，这里如果堆积消息过多，会访问磁盘 + // 另外如果频繁调用，是否会引起gc问题，需要关注 TODO + MessageExt msgExt = + this.brokerController.getMessageStore().lookMessageByOffset(requestHeader.getOffset()); + if (null == msgExt) { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("look message by offset failed, " + requestHeader.getOffset()); + return response; + } + + // 构造消息 + final String retryTopic = msgExt.getProperty(MessageConst.PROPERTY_RETRY_TOPIC); + if (null == retryTopic) { + MessageAccessor.putProperty(msgExt, MessageConst.PROPERTY_RETRY_TOPIC, msgExt.getTopic()); + } + msgExt.setWaitStoreMsgOK(false); + + // 客户端自动决定定时级别 + int delayLevel = requestHeader.getDelayLevel(); + + // 死信消息处理 + if (msgExt.getReconsumeTimes() >= subscriptionGroupConfig.getRetryMaxTimes()// + || delayLevel < 0) { + newTopic = MixAll.getDLQTopic(requestHeader.getGroup()); + queueIdInt = Math.abs(this.random.nextInt() % 99999999) % DLQ_NUMS_PER_GROUP; + + topicConfig = + this.brokerController.getTopicConfigManager().createTopicInSendMessageBackMethod( + newTopic, // + DLQ_NUMS_PER_GROUP,// + PermName.PERM_WRITE, 0 // 死信消息不需要同步，不需要较正。 + ); + if (null == topicConfig) { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("topic[" + newTopic + "] not exist"); + return response; + } + } + // 继续重试 + else { + if (0 == delayLevel) { + delayLevel = 3 + msgExt.getReconsumeTimes(); + } + + msgExt.setDelayTimeLevel(delayLevel); + } + + MessageExtBrokerInner msgInner = new MessageExtBrokerInner(); + msgInner.setTopic(newTopic); + msgInner.setBody(msgExt.getBody()); + msgInner.setFlag(msgExt.getFlag()); + MessageAccessor.setProperties(msgInner, msgExt.getProperties()); + msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgExt.getProperties())); + msgInner.setTagsCode(MessageExtBrokerInner.tagsString2tagsCode(null, msgExt.getTags())); + + msgInner.setQueueId(queueIdInt); + msgInner.setSysFlag(msgExt.getSysFlag()); + msgInner.setBornTimestamp(msgExt.getBornTimestamp()); + msgInner.setBornHost(msgExt.getBornHost()); + msgInner.setStoreHost(this.getStoreHost()); + msgInner.setReconsumeTimes(msgExt.getReconsumeTimes() + 1); + + // 保存源生消息的 msgId + String originMsgId = MessageAccessor.getOriginMessageId(msgExt); + MessageAccessor.setOriginMessageId(msgInner, UtilAll.isBlank(originMsgId) ? msgExt.getMsgId() + : originMsgId); + + PutMessageResult putMessageResult = this.brokerController.getMessageStore().putMessage(msgInner); + if (putMessageResult != null) { + switch (putMessageResult.getPutMessageStatus()) { + case PUT_OK: + // 统计失败重试的Topic + String backTopic = msgExt.getTopic(); + String correctTopic = msgExt.getProperty(MessageConst.PROPERTY_RETRY_TOPIC); + if (correctTopic != null) { + backTopic = correctTopic; + } + + this.brokerController.getBrokerStatsManager().incSendBackNums(requestHeader.getGroup(), + backTopic); + + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + + return response; + default: + break; + } + + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark(putMessageResult.getPutMessageStatus().name()); + return response; + } + + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("putMessageResult is null"); + return response; + } + + + private String diskUtil() { + String storePathPhysic = this.brokerController.getMessageStoreConfig().getStorePathCommitLog(); + double physicRatio = UtilAll.getDiskPartitionSpaceUsedPercent(storePathPhysic); + + String storePathLogis = + StorePathConfigHelper.getStorePathConsumeQueue(this.brokerController.getMessageStoreConfig() + .getStorePathRootDir()); + double logisRatio = UtilAll.getDiskPartitionSpaceUsedPercent(storePathLogis); + + String storePathIndex = + StorePathConfigHelper.getStorePathIndex(this.brokerController.getMessageStoreConfig() + .getStorePathRootDir()); + double indexRatio = UtilAll.getDiskPartitionSpaceUsedPercent(storePathIndex); + + return String.format("CL: %5.2f CQ: %5.2f INDEX: %5.2f", physicRatio, logisRatio, indexRatio); + } + + + private RemotingCommand sendMessage(final ChannelHandlerContext ctx, // + final RemotingCommand request,// + final SendMessageContext mqtraceContext,// + final SendMessageRequestHeader requestHeader) throws RemotingCommandException { + + final RemotingCommand response = + RemotingCommand.createResponseCommand(SendMessageResponseHeader.class); + final SendMessageResponseHeader responseHeader = + (SendMessageResponseHeader) response.readCustomHeader(); + + // 由于有直接返回的逻辑，所以必须要设置 + response.setOpaque(request.getOpaque()); + + if (log.isDebugEnabled()) { + log.debug("receive SendMessage request command, " + request); + } + + // 检查Broker权限, 顺序消息禁写；非顺序消息通过 nameserver 通知客户端剔除禁写分区 + if (!PermName.isWriteable(this.brokerController.getBrokerConfig().getBrokerPermission()) + && this.brokerController.getTopicConfigManager().isOrderTopic(requestHeader.getTopic())) { + response.setCode(ResponseCode.NO_PERMISSION); + response.setRemark("the broker[" + this.brokerController.getBrokerConfig().getBrokerIP1() + + "] sending message is forbidden"); + return response; + } + + final byte[] body = request.getBody(); + + // Topic名字是否与保留字段冲突 + if (!this.brokerController.getTopicConfigManager().isTopicCanSendMessage(requestHeader.getTopic())) { + String errorMsg = + "the topic[" + requestHeader.getTopic() + "] is conflict with system reserved words."; + log.warn(errorMsg); + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark(errorMsg); + return response; + } + + // 检查topic是否存在 + TopicConfig topicConfig = + this.brokerController.getTopicConfigManager().selectTopicConfig(requestHeader.getTopic()); + if (null == topicConfig) { + // 如果是单元化模式，则对 topic 进行设置 + int topicSysFlag = 0; + if (requestHeader.isUnitMode()) { + if (requestHeader.getTopic().startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) { + topicSysFlag = TopicSysFlag.buildSysFlag(false, true); + } + else { + topicSysFlag = TopicSysFlag.buildSysFlag(true, false); + } + } + + log.warn("the topic " + requestHeader.getTopic() + " not exist, producer: " + + ctx.channel().remoteAddress()); + topicConfig = this.brokerController.getTopicConfigManager().createTopicInSendMessageMethod(// + requestHeader.getTopic(), // + requestHeader.getDefaultTopic(), // + RemotingHelper.parseChannelRemoteAddr(ctx.channel()), // + requestHeader.getDefaultTopicQueueNums(), topicSysFlag); + + // 尝试看下是否是失败消息发回 + if (null == topicConfig) { + if (requestHeader.getTopic().startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) { + topicConfig = + this.brokerController.getTopicConfigManager().createTopicInSendMessageBackMethod( + requestHeader.getTopic(), 1, PermName.PERM_WRITE | PermName.PERM_READ, + topicSysFlag); + } + } + + if (null == topicConfig) { + response.setCode(ResponseCode.TOPIC_NOT_EXIST); + response.setRemark("topic[" + requestHeader.getTopic() + "] not exist, apply first please!" + + FAQUrl.suggestTodo(FAQUrl.APPLY_TOPIC_URL)); + return response; + } + } + + /** + * Broker本身不做Topic的权限验证，由Name Server负责通知Client处理 + */ + // // 检查topic权限 + // if (!PermName.isWriteable(topicConfig.getPerm())) { + // response.setCode(ResponseCode.NO_PERMISSION); + // response.setRemark("the topic[" + requestHeader.getOriginTopic() + + // "] sending message is forbidden"); + // return response; + // } + + // 检查队列有效性 + int queueIdInt = requestHeader.getQueueId(); + int idValid = Math.max(topicConfig.getWriteQueueNums(), topicConfig.getReadQueueNums()); + if (queueIdInt >= idValid) { + String errorInfo = String.format("request queueId[%d] is illagal, %s Producer: %s",// + queueIdInt,// + topicConfig.toString(),// + RemotingHelper.parseChannelRemoteAddr(ctx.channel())); + + log.warn(errorInfo); + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark(errorInfo); + + return response; + } + + // 随机指定一个队列 + if (queueIdInt < 0) { + queueIdInt = Math.abs(this.random.nextInt() % 99999999) % topicConfig.getWriteQueueNums(); + } + + int sysFlag = requestHeader.getSysFlag(); + // 多标签过滤需要置位 + if (TopicFilterType.MULTI_TAG == topicConfig.getTopicFilterType()) { + sysFlag |= MessageSysFlag.MultiTagsFlag; + } + + MessageExtBrokerInner msgInner = new MessageExtBrokerInner(); + msgInner.setTopic(requestHeader.getTopic()); + msgInner.setBody(body); + msgInner.setFlag(requestHeader.getFlag()); + MessageAccessor.setProperties(msgInner, + MessageDecoder.string2messageProperties(requestHeader.getProperties())); + msgInner.setPropertiesString(requestHeader.getProperties()); + msgInner.setTagsCode(MessageExtBrokerInner.tagsString2tagsCode(topicConfig.getTopicFilterType(), + msgInner.getTags())); + + msgInner.setQueueId(queueIdInt); + msgInner.setSysFlag(sysFlag); + msgInner.setBornTimestamp(requestHeader.getBornTimestamp()); + msgInner.setBornHost(ctx.channel().remoteAddress()); + msgInner.setStoreHost(this.getStoreHost()); + msgInner.setReconsumeTimes(requestHeader.getReconsumeTimes() == null ? 0 : requestHeader + .getReconsumeTimes()); + + // 检查事务消息 + if (this.brokerController.getBrokerConfig().isRejectTransactionMessage()) { + String traFlag = msgInner.getProperty(MessageConst.PROPERTY_TRANSACTION_PREPARED); + if (traFlag != null) { + response.setCode(ResponseCode.NO_PERMISSION); + response.setRemark("the broker[" + this.brokerController.getBrokerConfig().getBrokerIP1() + + "] sending transaction message is forbidden"); + return response; + } + } + + PutMessageResult putMessageResult = this.brokerController.getMessageStore().putMessage(msgInner); + if (putMessageResult != null) { + boolean sendOK = false; + + switch (putMessageResult.getPutMessageStatus()) { + // Success + case PUT_OK: + sendOK = true; + response.setCode(ResponseCode.SUCCESS); + break; + case FLUSH_DISK_TIMEOUT: + response.setCode(ResponseCode.FLUSH_DISK_TIMEOUT); + sendOK = true; + break; + case FLUSH_SLAVE_TIMEOUT: + response.setCode(ResponseCode.FLUSH_SLAVE_TIMEOUT); + sendOK = true; + break; + case SLAVE_NOT_AVAILABLE: + response.setCode(ResponseCode.SLAVE_NOT_AVAILABLE); + sendOK = true; + break; + + // Failed + case CREATE_MAPEDFILE_FAILED: + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("create maped file failed, please make sure OS and JDK both 64bit."); + break; + case MESSAGE_ILLEGAL: + response.setCode(ResponseCode.MESSAGE_ILLEGAL); + response.setRemark("the message is illegal, maybe length not matched."); + break; + case SERVICE_NOT_AVAILABLE: + response.setCode(ResponseCode.SERVICE_NOT_AVAILABLE); + response.setRemark("service not available now, maybe disk full, " + diskUtil() + + ", maybe your broker machine memory too small."); + break; + case UNKNOWN_ERROR: + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("UNKNOWN_ERROR"); + break; + default: + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("UNKNOWN_ERROR DEFAULT"); + break; + } + + if (sendOK) { + // 统计 + this.brokerController.getBrokerStatsManager().incTopicPutNums(msgInner.getTopic()); + this.brokerController.getBrokerStatsManager().incTopicPutSize(msgInner.getTopic(), + putMessageResult.getAppendMessageResult().getWroteBytes()); + this.brokerController.getBrokerStatsManager().incBrokerPutNums(); + + response.setRemark(null); + + responseHeader.setMsgId(putMessageResult.getAppendMessageResult().getMsgId()); + responseHeader.setQueueId(queueIdInt); + responseHeader.setQueueOffset(putMessageResult.getAppendMessageResult().getLogicsOffset()); + + // 直接返回 + if (!request.isOnewayRPC()) { + try { + ctx.writeAndFlush(response); + } + catch (Throwable e) { + log.error("SendMessageProcessor process request over, but response failed", e); + log.error(request.toString()); + log.error(response.toString()); + } + } + + if (this.brokerController.getBrokerConfig().isLongPollingEnable()) { + this.brokerController.getPullRequestHoldService().notifyMessageArriving( + requestHeader.getTopic(), queueIdInt, + putMessageResult.getAppendMessageResult().getLogicsOffset() + 1); + } + + // 消息轨迹：记录发送成功的消息 + if (hasSendMessageHook()) { + mqtraceContext.setMsgId(responseHeader.getMsgId()); + mqtraceContext.setQueueId(responseHeader.getQueueId()); + mqtraceContext.setQueueOffset(responseHeader.getQueueOffset()); + } + return null; + } + } + else { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("store putMessage return null"); + } + + return response; + } + + + public SocketAddress getStoreHost() { + return storeHost; + } + + /** + * 发送每条消息会回调 + */ + private List sendMessageHookList; + + + public boolean hasSendMessageHook() { + return sendMessageHookList != null && !this.sendMessageHookList.isEmpty(); + } + + + public void registerSendMessageHook(List sendMessageHookList) { + this.sendMessageHookList = sendMessageHookList; + } + + + public void executeSendMessageHookBefore(final ChannelHandlerContext ctx, final RemotingCommand request, + SendMessageContext context) { + if (hasSendMessageHook()) { + for (SendMessageHook hook : this.sendMessageHookList) { + try { + final SendMessageRequestHeader requestHeader = + (SendMessageRequestHeader) request + .decodeCommandCustomHeader(SendMessageRequestHeader.class); + context.setProducerGroup(requestHeader.getProducerGroup()); + context.setTopic(requestHeader.getTopic()); + context.setBodyLength(request.getBody().length); + context.setMsgProps(requestHeader.getProperties()); + context.setBornHost(RemotingHelper.parseChannelRemoteAddr(ctx.channel())); + context.setBrokerAddr(this.brokerController.getBrokerAddr()); + context.setQueueId(requestHeader.getQueueId()); + hook.sendMessageBefore(context); + requestHeader.setProperties(context.getMsgProps()); + } + catch (Throwable e) { + } + } + } + } + + + public void executeSendMessageHookAfter(final RemotingCommand response, final SendMessageContext context) { + if (hasSendMessageHook()) { + for (SendMessageHook hook : this.sendMessageHookList) { + try { + if (response != null) { + final SendMessageResponseHeader responseHeader = + (SendMessageResponseHeader) response.readCustomHeader(); + context.setMsgId(responseHeader.getMsgId()); + context.setQueueId(responseHeader.getQueueId()); + context.setQueueOffset(responseHeader.getQueueOffset()); + context.setCode(response.getCode()); + context.setErrorMsg(response.getRemark()); + } + hook.sendMessageAfter(context); + } + catch (Throwable e) { + } + } + } + } + + /** + * 消费每条消息会回调 + */ + private List consumeMessageHookList; + + + public boolean hasConsumeMessageHook() { + return consumeMessageHookList != null && !this.consumeMessageHookList.isEmpty(); + } + + + public void registerConsumeMessageHook(List consumeMessageHookList) { + this.consumeMessageHookList = consumeMessageHookList; + } + + + public void executeConsumeMessageHookAfter(final ConsumeMessageContext context) { + if (hasConsumeMessageHook()) { + for (ConsumeMessageHook hook : this.consumeMessageHookList) { + try { + hook.consumeMessageAfter(context); + } + catch (Throwable e) { + } + } + } + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/slave/SlaveSynchronize.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/slave/SlaveSynchronize.java new file mode 100644 index 000000000..6603cab87 --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/slave/SlaveSynchronize.java @@ -0,0 +1,166 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.broker.slave; + +import java.io.IOException; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.rocketmq.broker.BrokerController; +import com.alibaba.rocketmq.broker.subscription.SubscriptionGroupManager; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.common.protocol.body.ConsumerOffsetSerializeWrapper; +import com.alibaba.rocketmq.common.protocol.body.SubscriptionGroupWrapper; +import com.alibaba.rocketmq.common.protocol.body.TopicConfigSerializeWrapper; +import com.alibaba.rocketmq.store.config.StorePathConfigHelper; + + +/** + * Slave从Master同步信息（非消息） + * + * @author shijia.wxr + * @author manhong.yqd + * @since 2013-7-8 + */ +public class SlaveSynchronize { + private static final Logger log = LoggerFactory.getLogger(LoggerName.BrokerLoggerName); + private final BrokerController brokerController; + private volatile String masterAddr = null; + + + public SlaveSynchronize(BrokerController brokerController) { + this.brokerController = brokerController; + } + + + public String getMasterAddr() { + return masterAddr; + } + + + public void setMasterAddr(String masterAddr) { + this.masterAddr = masterAddr; + } + + + public void syncAll() { + this.syncTopicConfig(); + this.syncConsumerOffset(); + this.syncDelayOffset(); + this.syncSubscriptionGroupConfig(); + } + + + private void syncTopicConfig() { + String masterAddrBak = this.masterAddr; + if (masterAddrBak != null) { + try { + TopicConfigSerializeWrapper topicWrapper = + this.brokerController.getBrokerOuterAPI().getAllTopicConfig(masterAddrBak); + if (!this.brokerController.getTopicConfigManager().getDataVersion() + .equals(topicWrapper.getDataVersion())) { + + this.brokerController.getTopicConfigManager().getDataVersion() + .assignNewOne(topicWrapper.getDataVersion()); + this.brokerController.getTopicConfigManager().getTopicConfigTable().clear(); + this.brokerController.getTopicConfigManager().getTopicConfigTable() + .putAll(topicWrapper.getTopicConfigTable()); + this.brokerController.getTopicConfigManager().persist(); + + log.info("update slave topic config from master, {}", masterAddrBak); + } + } + catch (Exception e) { + log.error("syncTopicConfig Exception, " + masterAddrBak, e); + } + } + } + + + private void syncConsumerOffset() { + String masterAddrBak = this.masterAddr; + if (masterAddrBak != null) { + try { + ConsumerOffsetSerializeWrapper offsetWrapper = + this.brokerController.getBrokerOuterAPI().getAllConsumerOffset(masterAddrBak); + this.brokerController.getConsumerOffsetManager().getOffsetTable() + .putAll(offsetWrapper.getOffsetTable()); + this.brokerController.getConsumerOffsetManager().persist(); + log.info("update slave consumer offset from master, {}", masterAddrBak); + } + catch (Exception e) { + log.error("syncConsumerOffset Exception, " + masterAddrBak, e); + } + } + } + + + private void syncDelayOffset() { + String masterAddrBak = this.masterAddr; + if (masterAddrBak != null) { + try { + String delayOffset = + this.brokerController.getBrokerOuterAPI().getAllDelayOffset(masterAddrBak); + if (delayOffset != null) { + + String fileName = + StorePathConfigHelper.getDelayOffsetStorePath(this.brokerController + .getMessageStoreConfig().getStorePathRootDir()); + try { + MixAll.string2File(delayOffset, fileName); + } + catch (IOException e) { + log.error("persist file Exception, " + fileName, e); + } + } + log.info("update slave delay offset from master, {}", masterAddrBak); + } + catch (Exception e) { + log.error("syncDelayOffset Exception, " + masterAddrBak, e); + } + } + } + + + private void syncSubscriptionGroupConfig() { + String masterAddrBak = this.masterAddr; + if (masterAddrBak != null) { + try { + SubscriptionGroupWrapper subscriptionWrapper = + this.brokerController.getBrokerOuterAPI() + .getAllSubscriptionGroupConfig(masterAddrBak); + + if (!this.brokerController.getSubscriptionGroupManager().getDataVersion() + .equals(subscriptionWrapper.getDataVersion())) { + SubscriptionGroupManager subscriptionGroupManager = + this.brokerController.getSubscriptionGroupManager(); + subscriptionGroupManager.getDataVersion().assignNewOne( + subscriptionWrapper.getDataVersion()); + subscriptionGroupManager.getSubscriptionGroupTable().clear(); + subscriptionGroupManager.getSubscriptionGroupTable().putAll( + subscriptionWrapper.getSubscriptionGroupTable()); + subscriptionGroupManager.persist(); + log.info("update slave Subscription Group from master, {}", masterAddrBak); + } + } + catch (Exception e) { + log.error("syncSubscriptionGroup Exception, " + masterAddrBak, e); + } + } + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/subscription/SubscriptionGroupManager.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/subscription/SubscriptionGroupManager.java new file mode 100644 index 000000000..0baef873e --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/subscription/SubscriptionGroupManager.java @@ -0,0 +1,172 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.broker.subscription; + +import java.util.Iterator; +import java.util.Map.Entry; +import java.util.concurrent.ConcurrentHashMap; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.rocketmq.broker.BrokerController; +import com.alibaba.rocketmq.broker.BrokerPathConfigHelper; +import com.alibaba.rocketmq.common.ConfigManager; +import com.alibaba.rocketmq.common.DataVersion; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.common.subscription.SubscriptionGroupConfig; +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; + + +/** + * 用来管理订阅组，包括订阅权限等 + * + * @author shijia.wxr + * @since 2013-7-26 + */ +public class SubscriptionGroupManager extends ConfigManager { + private static final Logger log = LoggerFactory.getLogger(LoggerName.BrokerLoggerName); + private transient BrokerController brokerController; + + // 订阅组 + private final ConcurrentHashMap subscriptionGroupTable = + new ConcurrentHashMap(1024); + private final DataVersion dataVersion = new DataVersion(); + + + private void init() { + { + SubscriptionGroupConfig subscriptionGroupConfig = new SubscriptionGroupConfig(); + subscriptionGroupConfig.setGroupName(MixAll.TOOLS_CONSUMER_GROUP); + this.subscriptionGroupTable.put(MixAll.TOOLS_CONSUMER_GROUP, subscriptionGroupConfig); + } + + { + SubscriptionGroupConfig subscriptionGroupConfig = new SubscriptionGroupConfig(); + subscriptionGroupConfig.setGroupName(MixAll.FILTERSRV_CONSUMER_GROUP); + this.subscriptionGroupTable.put(MixAll.FILTERSRV_CONSUMER_GROUP, subscriptionGroupConfig); + } + } + + + public SubscriptionGroupManager() { + this.init(); + } + + + public SubscriptionGroupManager(BrokerController brokerController) { + this.brokerController = brokerController; + this.init(); + } + + + public void updateSubscriptionGroupConfig(final SubscriptionGroupConfig config) { + SubscriptionGroupConfig old = this.subscriptionGroupTable.put(config.getGroupName(), config); + if (old != null) { + log.info("update subscription group config, old: " + old + " new: " + config); + } + else { + log.info("create new subscription group, " + config); + } + + this.dataVersion.nextVersion(); + + this.persist(); + } + + + public SubscriptionGroupConfig findSubscriptionGroupConfig(final String group) { + SubscriptionGroupConfig subscriptionGroupConfig = this.subscriptionGroupTable.get(group); + if (null == subscriptionGroupConfig) { + if (brokerController.getBrokerConfig().isAutoCreateSubscriptionGroup()) { + subscriptionGroupConfig = new SubscriptionGroupConfig(); + subscriptionGroupConfig.setGroupName(group); + this.subscriptionGroupTable.putIfAbsent(group, subscriptionGroupConfig); + log.info("auto create a subscription group, {}", subscriptionGroupConfig.toString()); + this.dataVersion.nextVersion(); + this.persist(); + } + } + + return subscriptionGroupConfig; + } + + + @Override + public String encode() { + return this.encode(false); + } + + + public String encode(final boolean prettyFormat) { + return RemotingSerializable.toJson(this, prettyFormat); + } + + + @Override + public void decode(String jsonString) { + if (jsonString != null) { + SubscriptionGroupManager obj = + RemotingSerializable.fromJson(jsonString, SubscriptionGroupManager.class); + if (obj != null) { + this.subscriptionGroupTable.putAll(obj.subscriptionGroupTable); + this.dataVersion.assignNewOne(obj.dataVersion); + this.printLoadDataWhenFirstBoot(obj); + } + } + } + + + private void printLoadDataWhenFirstBoot(final SubscriptionGroupManager sgm) { + Iterator> it = + sgm.getSubscriptionGroupTable().entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + log.info("load exist subscription group, {}", next.getValue().toString()); + } + } + + + @Override + public String configFilePath() { + return BrokerPathConfigHelper.getSubscriptionGroupPath(this.brokerController.getMessageStoreConfig() + .getStorePathRootDir()); + } + + + public ConcurrentHashMap getSubscriptionGroupTable() { + return subscriptionGroupTable; + } + + + public DataVersion getDataVersion() { + return dataVersion; + } + + + public void deleteSubscriptionGroupConfig(final String groupName) { + SubscriptionGroupConfig old = this.subscriptionGroupTable.remove(groupName); + if (old != null) { + log.info("delete subscription group OK, subscription group: " + old); + this.dataVersion.nextVersion(); + this.persist(); + } + else { + log.warn("delete subscription group failed, subscription group: " + old + " not exist"); + } + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/topic/TopicConfigManager.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/topic/TopicConfigManager.java new file mode 100644 index 000000000..38f257c96 --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/topic/TopicConfigManager.java @@ -0,0 +1,473 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.broker.topic; + +import java.util.HashSet; +import java.util.Iterator; +import java.util.Map.Entry; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.rocketmq.broker.BrokerController; +import com.alibaba.rocketmq.broker.BrokerPathConfigHelper; +import com.alibaba.rocketmq.common.ConfigManager; +import com.alibaba.rocketmq.common.DataVersion; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.TopicConfig; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.common.constant.PermName; +import com.alibaba.rocketmq.common.protocol.body.KVTable; +import com.alibaba.rocketmq.common.protocol.body.TopicConfigSerializeWrapper; +import com.alibaba.rocketmq.common.sysflag.TopicSysFlag; + + +/** + * Topic配置管理 + * + * @author shijia.wxr + * @author lansheng.zj@taobao.com + * @since 2013-7-26 + */ +public class TopicConfigManager extends ConfigManager { + private static final Logger log = LoggerFactory.getLogger(LoggerName.BrokerLoggerName); + private static final long LockTimeoutMillis = 3000; + private transient final Lock lockTopicConfigTable = new ReentrantLock(); + private transient BrokerController brokerController; + + // Topic配置 + private final ConcurrentHashMap topicConfigTable = + new ConcurrentHashMap(1024); + private final DataVersion dataVersion = new DataVersion(); + + private final Set systemTopicList = new HashSet(); + + + public TopicConfigManager() { + } + + + public TopicConfigManager(BrokerController brokerController) { + this.brokerController = brokerController; + { + // MixAll.SELF_TEST_TOPIC + String topic = MixAll.SELF_TEST_TOPIC; + TopicConfig topicConfig = new TopicConfig(topic); + this.systemTopicList.add(topic); + topicConfig.setReadQueueNums(1); + topicConfig.setWriteQueueNums(1); + this.topicConfigTable.put(topicConfig.getTopicName(), topicConfig); + } + { + // MixAll.DEFAULT_TOPIC + if (this.brokerController.getBrokerConfig().isAutoCreateTopicEnable()) { + String topic = MixAll.DEFAULT_TOPIC; + TopicConfig topicConfig = new TopicConfig(topic); + this.systemTopicList.add(topic); + topicConfig.setReadQueueNums(this.brokerController.getBrokerConfig() + .getDefaultTopicQueueNums()); + topicConfig.setWriteQueueNums(this.brokerController.getBrokerConfig() + .getDefaultTopicQueueNums()); + int perm = PermName.PERM_INHERIT | PermName.PERM_READ | PermName.PERM_WRITE; + topicConfig.setPerm(perm); + this.topicConfigTable.put(topicConfig.getTopicName(), topicConfig); + } + } + { + // MixAll.BENCHMARK_TOPIC + String topic = MixAll.BENCHMARK_TOPIC; + TopicConfig topicConfig = new TopicConfig(topic); + this.systemTopicList.add(topic); + topicConfig.setReadQueueNums(1024); + topicConfig.setWriteQueueNums(1024); + this.topicConfigTable.put(topicConfig.getTopicName(), topicConfig); + } + { + // 集群名字 + String topic = this.brokerController.getBrokerConfig().getBrokerClusterName(); + TopicConfig topicConfig = new TopicConfig(topic); + this.systemTopicList.add(topic); + int perm = PermName.PERM_INHERIT; + if (this.brokerController.getBrokerConfig().isClusterTopicEnable()) { + perm |= PermName.PERM_READ | PermName.PERM_WRITE; + } + topicConfig.setPerm(perm); + this.topicConfigTable.put(topicConfig.getTopicName(), topicConfig); + } + { + // 服务器名字 + String topic = this.brokerController.getBrokerConfig().getBrokerName(); + TopicConfig topicConfig = new TopicConfig(topic); + this.systemTopicList.add(topic); + int perm = PermName.PERM_INHERIT; + if (this.brokerController.getBrokerConfig().isBrokerTopicEnable()) { + perm |= PermName.PERM_READ | PermName.PERM_WRITE; + } + topicConfig.setReadQueueNums(1); + topicConfig.setWriteQueueNums(1); + topicConfig.setPerm(perm); + this.topicConfigTable.put(topicConfig.getTopicName(), topicConfig); + } + { + // MixAll.OFFSET_MOVED_EVENT + String topic = MixAll.OFFSET_MOVED_EVENT; + TopicConfig topicConfig = new TopicConfig(topic); + this.systemTopicList.add(topic); + topicConfig.setReadQueueNums(1); + topicConfig.setWriteQueueNums(1); + this.topicConfigTable.put(topicConfig.getTopicName(), topicConfig); + } + } + + + public boolean isSystemTopic(final String topic) { + return this.systemTopicList.contains(topic); + } + + + public Set getSystemTopic() { + return this.systemTopicList; + } + + + public boolean isTopicCanSendMessage(final String topic) { + boolean reservedWords = + topic.equals(MixAll.DEFAULT_TOPIC) + || topic.equals(this.brokerController.getBrokerConfig().getBrokerClusterName()); + + return !reservedWords; + } + + + public TopicConfig selectTopicConfig(final String topic) { + return this.topicConfigTable.get(topic); + } + + + /** + * 发消息时，如果Topic不存在，尝试创建 + */ + public TopicConfig createTopicInSendMessageMethod(final String topic, final String defaultTopic, + final String remoteAddress, final int clientDefaultTopicQueueNums, final int topicSysFlag) { + TopicConfig topicConfig = null; + boolean createNew = false; + + try { + if (this.lockTopicConfigTable.tryLock(LockTimeoutMillis, TimeUnit.MILLISECONDS)) { + try { + topicConfig = this.topicConfigTable.get(topic); + if (topicConfig != null) + return topicConfig; + + TopicConfig defaultTopicConfig = this.topicConfigTable.get(defaultTopic); + if (defaultTopicConfig != null) { + if (PermName.isInherited(defaultTopicConfig.getPerm())) { + topicConfig = new TopicConfig(topic); + + int queueNums = + clientDefaultTopicQueueNums > defaultTopicConfig.getWriteQueueNums() ? defaultTopicConfig + .getWriteQueueNums() : clientDefaultTopicQueueNums; + + if (queueNums < 0) { + queueNums = 0; + } + + topicConfig.setReadQueueNums(queueNums); + topicConfig.setWriteQueueNums(queueNums); + int perm = defaultTopicConfig.getPerm(); + perm &= ~PermName.PERM_INHERIT; + topicConfig.setPerm(perm); + topicConfig.setTopicSysFlag(topicSysFlag); + topicConfig.setTopicFilterType(defaultTopicConfig.getTopicFilterType()); + } + else { + log.warn("create new topic failed, because the default topic[" + defaultTopic + + "] no perm, " + defaultTopicConfig.getPerm() + " producer: " + + remoteAddress); + } + } + else { + log.warn("create new topic failed, because the default topic[" + defaultTopic + + "] not exist." + " producer: " + remoteAddress); + } + + if (topicConfig != null) { + log.info("create new topic by default topic[" + defaultTopic + "], " + topicConfig + + " producer: " + remoteAddress); + + this.topicConfigTable.put(topic, topicConfig); + + this.dataVersion.nextVersion(); + + createNew = true; + + this.persist(); + } + } + finally { + this.lockTopicConfigTable.unlock(); + } + } + } + catch (InterruptedException e) { + log.error("createTopicInSendMessageMethod exception", e); + } + + if (createNew) { + this.brokerController.registerBrokerAll(false); + } + + return topicConfig; + } + + + public TopicConfig createTopicInSendMessageBackMethod(// + final String topic, // + final int clientDefaultTopicQueueNums,// + final int perm,// + final int topicSysFlag) { + TopicConfig topicConfig = this.topicConfigTable.get(topic); + if (topicConfig != null) + return topicConfig; + + boolean createNew = false; + + try { + if (this.lockTopicConfigTable.tryLock(LockTimeoutMillis, TimeUnit.MILLISECONDS)) { + try { + topicConfig = this.topicConfigTable.get(topic); + if (topicConfig != null) + return topicConfig; + + topicConfig = new TopicConfig(topic); + topicConfig.setReadQueueNums(clientDefaultTopicQueueNums); + topicConfig.setWriteQueueNums(clientDefaultTopicQueueNums); + topicConfig.setPerm(perm); + topicConfig.setTopicSysFlag(topicSysFlag); + + log.info("create new topic {}", topicConfig); + this.topicConfigTable.put(topic, topicConfig); + createNew = true; + this.dataVersion.nextVersion(); + this.persist(); + } + finally { + this.lockTopicConfigTable.unlock(); + } + } + } + catch (InterruptedException e) { + log.error("createTopicInSendMessageBackMethod exception", e); + } + + if (createNew) { + this.brokerController.registerBrokerAll(false); + } + + return topicConfig; + } + + + /** + * 更新 topic 的单元化标识 + */ + public void updateTopicUnitFlag(final String topic, final boolean unit) { + + TopicConfig topicConfig = this.topicConfigTable.get(topic); + if (topicConfig != null) { + int oldTopicSysFlag = topicConfig.getTopicSysFlag(); + if (unit) { + topicConfig.setTopicSysFlag(TopicSysFlag.setUnitFlag(oldTopicSysFlag)); + } + else { + topicConfig.setTopicSysFlag(TopicSysFlag.clearUnitFlag(oldTopicSysFlag)); + } + + log.info("update topic sys flag. oldTopicSysFlag={}, newTopicSysFlag", oldTopicSysFlag, + topicConfig.getTopicSysFlag()); + + this.topicConfigTable.put(topic, topicConfig); + + this.dataVersion.nextVersion(); + + this.persist(); + this.brokerController.registerBrokerAll(false); + } + } + + + /** + * 更新 topic 是否有单元化订阅组 + */ + public void updateTopicUnitSubFlag(final String topic, final boolean hasUnitSub) { + TopicConfig topicConfig = this.topicConfigTable.get(topic); + if (topicConfig != null) { + int oldTopicSysFlag = topicConfig.getTopicSysFlag(); + if (hasUnitSub) { + topicConfig.setTopicSysFlag(TopicSysFlag.setUnitSubFlag(oldTopicSysFlag)); + } + + log.info("update topic sys flag. oldTopicSysFlag={}, newTopicSysFlag", oldTopicSysFlag, + topicConfig.getTopicSysFlag()); + + this.topicConfigTable.put(topic, topicConfig); + + this.dataVersion.nextVersion(); + + this.persist(); + this.brokerController.registerBrokerAll(false); + } + } + + + public void updateTopicConfig(final TopicConfig topicConfig) { + TopicConfig old = this.topicConfigTable.put(topicConfig.getTopicName(), topicConfig); + if (old != null) { + log.info("update topic config, old: " + old + " new: " + topicConfig); + } + else { + log.info("create new topic, " + topicConfig); + } + + this.dataVersion.nextVersion(); + + this.brokerController.registerBrokerAll(false); + + this.persist(); + } + + + public void updateOrderTopicConfig(final KVTable orderKVTableFromNs) { + // 根据 nameserver 上的 topic 配置同步检查更新 topic config 的顺序消息配置 + if (orderKVTableFromNs != null && orderKVTableFromNs.getTable() != null) { + boolean isChange = false; + Set orderTopics = orderKVTableFromNs.getTable().keySet(); + for (String topic : orderTopics) { + TopicConfig topicConfig = this.topicConfigTable.get(topic); + if (topicConfig != null && !topicConfig.isOrder()) { + topicConfig.setOrder(true); + isChange = true; + log.info("update order topic config, topic={}, order={}", topic, true); + } + } + for (String topic : this.topicConfigTable.keySet()) { + if (!orderTopics.contains(topic)) { + TopicConfig topicConfig = this.topicConfigTable.get(topic); + if (topicConfig.isOrder()) { + topicConfig.setOrder(false); + isChange = true; + log.info("update order topic config, topic={}, order={}", topic, false); + } + } + } + if (isChange) { + this.dataVersion.nextVersion(); + this.persist(); + } + } + } + + + public boolean isOrderTopic(final String topic) { + TopicConfig topicConfig = this.topicConfigTable.get(topic); + if (topicConfig == null) { + return false; + } + else { + return topicConfig.isOrder(); + } + } + + + public void deleteTopicConfig(final String topic) { + TopicConfig old = this.topicConfigTable.remove(topic); + if (old != null) { + log.info("delete topic config OK, topic: " + old); + this.dataVersion.nextVersion(); + this.persist(); + } + else { + log.warn("delete topic config failed, topic: " + topic + " not exist"); + } + } + + + public TopicConfigSerializeWrapper buildTopicConfigSerializeWrapper() { + TopicConfigSerializeWrapper topicConfigSerializeWrapper = new TopicConfigSerializeWrapper(); + topicConfigSerializeWrapper.setTopicConfigTable(this.topicConfigTable); + topicConfigSerializeWrapper.setDataVersion(this.dataVersion); + return topicConfigSerializeWrapper; + } + + + @Override + public String encode() { + return encode(false); + } + + + public String encode(final boolean prettyFormat) { + TopicConfigSerializeWrapper topicConfigSerializeWrapper = new TopicConfigSerializeWrapper(); + topicConfigSerializeWrapper.setTopicConfigTable(this.topicConfigTable); + topicConfigSerializeWrapper.setDataVersion(this.dataVersion); + return topicConfigSerializeWrapper.toJson(prettyFormat); + } + + + @Override + public void decode(String jsonString) { + if (jsonString != null) { + TopicConfigSerializeWrapper topicConfigSerializeWrapper = + TopicConfigSerializeWrapper.fromJson(jsonString, TopicConfigSerializeWrapper.class); + if (topicConfigSerializeWrapper != null) { + this.topicConfigTable.putAll(topicConfigSerializeWrapper.getTopicConfigTable()); + this.dataVersion.assignNewOne(topicConfigSerializeWrapper.getDataVersion()); + this.printLoadDataWhenFirstBoot(topicConfigSerializeWrapper); + } + } + } + + + private void printLoadDataWhenFirstBoot(final TopicConfigSerializeWrapper tcs) { + Iterator> it = tcs.getTopicConfigTable().entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + log.info("load exist local topic, {}", next.getValue().toString()); + } + } + + + @Override + public String configFilePath() { + return BrokerPathConfigHelper.getTopicConfigPath(this.brokerController.getMessageStoreConfig() + .getStorePathRootDir()); + } + + + public DataVersion getDataVersion() { + return dataVersion; + } + + + public ConcurrentHashMap getTopicConfigTable() { + return topicConfigTable; + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/transaction/TransactionRecord.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/transaction/TransactionRecord.java new file mode 100644 index 000000000..d867bccf9 --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/transaction/TransactionRecord.java @@ -0,0 +1,27 @@ +package com.alibaba.rocketmq.broker.transaction; + +public class TransactionRecord { + // Commit Log Offset + private long offset; + private String producerGroup; + + + public long getOffset() { + return offset; + } + + + public void setOffset(long offset) { + this.offset = offset; + } + + + public String getProducerGroup() { + return producerGroup; + } + + + public void setProducerGroup(String producerGroup) { + this.producerGroup = producerGroup; + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/transaction/TransactionStore.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/transaction/TransactionStore.java new file mode 100644 index 000000000..bda996974 --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/transaction/TransactionStore.java @@ -0,0 +1,32 @@ +package com.alibaba.rocketmq.broker.transaction; + +import java.util.List; + + +/** + * 事务存储接口，主要为分布式事务消息存储服务 + */ +public interface TransactionStore { + public boolean open(); + + + public void close(); + + + public boolean put(final List trs); + + + public void remove(final List pks); + + + public List traverse(final long pk, final int nums); + + + public long totalRecords(); + + + public long minPK(); + + + public long maxPK(); +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/transaction/jdbc/JDBCTransactionStore.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/transaction/jdbc/JDBCTransactionStore.java new file mode 100644 index 000000000..a65aa8032 --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/transaction/jdbc/JDBCTransactionStore.java @@ -0,0 +1,263 @@ +package com.alibaba.rocketmq.broker.transaction.jdbc; + +import java.net.URL; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Arrays; +import java.util.List; +import java.util.Properties; +import java.util.concurrent.atomic.AtomicLong; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.rocketmq.broker.transaction.TransactionRecord; +import com.alibaba.rocketmq.broker.transaction.TransactionStore; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.constant.LoggerName; + + +public class JDBCTransactionStore implements TransactionStore { + private static final Logger log = LoggerFactory.getLogger(LoggerName.TransactionLoggerName); + private final JDBCTransactionStoreConfig jdbcTransactionStoreConfig; + private Connection connection; + private AtomicLong totalRecordsValue = new AtomicLong(0); + + + public JDBCTransactionStore(JDBCTransactionStoreConfig jdbcTransactionStoreConfig) { + this.jdbcTransactionStoreConfig = jdbcTransactionStoreConfig; + } + + + private boolean loadDriver() { + try { + Class.forName(this.jdbcTransactionStoreConfig.getJdbcDriverClass()).newInstance(); + log.info("Loaded the appropriate driver, {}", + this.jdbcTransactionStoreConfig.getJdbcDriverClass()); + return true; + } + catch (Exception e) { + log.info("Loaded the appropriate driver Exception", e); + } + + return false; + } + + + private boolean computeTotalRecords() { + Statement statement = null; + ResultSet resultSet = null; + try { + statement = this.connection.createStatement(); + + resultSet = statement.executeQuery("select count(offset) as total from t_transaction"); + if (!resultSet.next()) { + log.warn("computeTotalRecords ResultSet is empty"); + return false; + } + + this.totalRecordsValue.set(resultSet.getLong(1)); + } + catch (Exception e) { + log.warn("computeTotalRecords Exception", e); + return false; + } + finally { + if (null != statement) { + try { + statement.close(); + } + catch (SQLException e) { + } + } + + if (null != resultSet) { + try { + resultSet.close(); + } + catch (SQLException e) { + } + } + } + + return true; + } + + + private String createTableSql() { + URL resource = JDBCTransactionStore.class.getClassLoader().getResource("transaction.sql"); + String fileContent = MixAll.file2String(resource); + return fileContent; + } + + + private boolean createDB() { + Statement statement = null; + try { + statement = this.connection.createStatement(); + + String sql = this.createTableSql(); + log.info("createDB SQL:\n {}", sql); + statement.execute(sql); + this.connection.commit(); + return true; + } + catch (Exception e) { + log.warn("createDB Exception", e); + return false; + } + finally { + if (null != statement) { + try { + statement.close(); + } + catch (SQLException e) { + } + } + } + } + + + @Override + public boolean open() { + if (this.loadDriver()) { + Properties props = new Properties(); + props.put("user", jdbcTransactionStoreConfig.getJdbcUser()); + props.put("password", jdbcTransactionStoreConfig.getJdbcPassword()); + + try { + this.connection = + DriverManager.getConnection(this.jdbcTransactionStoreConfig.getJdbcURL(), props); + + this.connection.setAutoCommit(false); + + // 如果表不存在，尝试初始化表 + if (!this.computeTotalRecords()) { + return this.createDB(); + } + + return true; + } + catch (SQLException e) { + log.info("Create JDBC Connection Exeption", e); + } + } + + return false; + } + + + @Override + public void close() { + try { + if (this.connection != null) { + this.connection.close(); + } + } + catch (SQLException e) { + } + } + + + private long updatedRows(int[] rows) { + long res = 0; + for (int i : rows) { + res += i; + } + + return res; + } + + + @Override + public void remove(List pks) { + PreparedStatement statement = null; + try { + this.connection.setAutoCommit(false); + statement = this.connection.prepareStatement("DELETE FROM t_transaction WHERE offset = ?"); + for (long pk : pks) { + statement.setLong(1, pk); + statement.addBatch(); + } + int[] executeBatch = statement.executeBatch(); + System.out.println(Arrays.toString(executeBatch)); + this.connection.commit(); + } + catch (Exception e) { + log.warn("createDB Exception", e); + } + finally { + if (null != statement) { + try { + statement.close(); + } + catch (SQLException e) { + } + } + } + } + + + @Override + public List traverse(long pk, int nums) { + // TODO Auto-generated method stub + return null; + } + + + @Override + public long totalRecords() { + // TODO Auto-generated method stub + return this.totalRecordsValue.get(); + } + + + @Override + public long minPK() { + // TODO Auto-generated method stub + return 0; + } + + + @Override + public long maxPK() { + // TODO Auto-generated method stub + return 0; + } + + + @Override + public boolean put(List trs) { + PreparedStatement statement = null; + try { + this.connection.setAutoCommit(false); + statement = this.connection.prepareStatement("insert into t_transaction values (?, ?)"); + for (TransactionRecord tr : trs) { + statement.setLong(1, tr.getOffset()); + statement.setString(2, tr.getProducerGroup()); + statement.addBatch(); + } + int[] executeBatch = statement.executeBatch(); + this.connection.commit(); + this.totalRecordsValue.addAndGet(updatedRows(executeBatch)); + return true; + } + catch (Exception e) { + log.warn("createDB Exception", e); + return false; + } + finally { + if (null != statement) { + try { + statement.close(); + } + catch (SQLException e) { + } + } + } + } +} diff --git a/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/transaction/jdbc/JDBCTransactionStoreConfig.java b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/transaction/jdbc/JDBCTransactionStoreConfig.java new file mode 100644 index 000000000..10a822b60 --- /dev/null +++ b/rocketmq-broker/src/main/java/com/alibaba/rocketmq/broker/transaction/jdbc/JDBCTransactionStoreConfig.java @@ -0,0 +1,48 @@ +package com.alibaba.rocketmq.broker.transaction.jdbc; + +public class JDBCTransactionStoreConfig { + private String jdbcDriverClass = "com.mysql.jdbc.Driver"; + private String jdbcURL = "jdbc:mysql://xxx.xxx.xxx.xxx:1000/xxx?useUnicode=true&characterEncoding=UTF-8"; + private String jdbcUser = "xxx"; + private String jdbcPassword = "xxx"; + + + public String getJdbcDriverClass() { + return jdbcDriverClass; + } + + + public void setJdbcDriverClass(String jdbcDriverClass) { + this.jdbcDriverClass = jdbcDriverClass; + } + + + public String getJdbcURL() { + return jdbcURL; + } + + + public void setJdbcURL(String jdbcURL) { + this.jdbcURL = jdbcURL; + } + + + public String getJdbcUser() { + return jdbcUser; + } + + + public void setJdbcUser(String jdbcUser) { + this.jdbcUser = jdbcUser; + } + + + public String getJdbcPassword() { + return jdbcPassword; + } + + + public void setJdbcPassword(String jdbcPassword) { + this.jdbcPassword = jdbcPassword; + } +} diff --git a/rocketmq-broker/src/main/resources/transaction.sql b/rocketmq-broker/src/main/resources/transaction.sql new file mode 100644 index 000000000..aaefe4335 --- /dev/null +++ b/rocketmq-broker/src/main/resources/transaction.sql @@ -0,0 +1,4 @@ +CREATE TABLE t_transaction( + offset NUMERIC(20) PRIMARY KEY, + producerGroup VARCHAR(64) +) diff --git a/rocketmq-broker/src/test/java/com/alibaba/rocketmq/broker/api/SendMessageTest.java b/rocketmq-broker/src/test/java/com/alibaba/rocketmq/broker/api/SendMessageTest.java new file mode 100644 index 000000000..d169acc03 --- /dev/null +++ b/rocketmq-broker/src/test/java/com/alibaba/rocketmq/broker/api/SendMessageTest.java @@ -0,0 +1,73 @@ +/** + * $Id: SendMessageTest.java 1831 2013-05-16 01:39:51Z shijia.wxr $ + */ +package com.alibaba.rocketmq.broker.api; + +import org.junit.Test; + +import com.alibaba.rocketmq.broker.BrokerController; +import com.alibaba.rocketmq.client.impl.CommunicationMode; +import com.alibaba.rocketmq.client.impl.MQClientAPIImpl; +import com.alibaba.rocketmq.client.producer.SendResult; +import com.alibaba.rocketmq.common.BrokerConfig; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.message.Message; +import com.alibaba.rocketmq.common.message.MessageDecoder; +import com.alibaba.rocketmq.common.protocol.header.SendMessageRequestHeader; +import com.alibaba.rocketmq.remoting.netty.NettyClientConfig; +import com.alibaba.rocketmq.remoting.netty.NettyServerConfig; +import com.alibaba.rocketmq.store.config.MessageStoreConfig; + + +/** + * @author shijia.wxr + */ +public class SendMessageTest { + @Test + public void test_sendMessage() throws Exception { + BrokerController brokerController = new BrokerController(// + new BrokerConfig(), // + new NettyServerConfig(), // + new NettyClientConfig(), // + new MessageStoreConfig()); + boolean initResult = brokerController.initialize(); + System.out.println("initialize " + initResult); + + brokerController.start(); + + MQClientAPIImpl client = new MQClientAPIImpl(new NettyClientConfig(), null); + client.start(); + + for (int i = 0; i < 100000; i++) { + String topic = "UnitTestTopic_" + i % 3; + Message msg = + new Message(topic, "TAG1 TAG2", "100200300", ("Hello, Nice world\t" + i).getBytes()); + msg.setDelayTimeLevel(i % 3 + 1); + + try { + SendMessageRequestHeader requestHeader = new SendMessageRequestHeader(); + requestHeader.setProducerGroup("abc"); + requestHeader.setTopic(msg.getTopic()); + requestHeader.setDefaultTopic(MixAll.DEFAULT_TOPIC); + requestHeader.setDefaultTopicQueueNums(4); + requestHeader.setQueueId(i % 4); + requestHeader.setSysFlag(0); + requestHeader.setBornTimestamp(System.currentTimeMillis()); + requestHeader.setFlag(msg.getFlag()); + requestHeader.setProperties(MessageDecoder.messageProperties2String(msg.getProperties())); + + SendResult result = + client.sendMessage("127.0.0.1:10911", "brokerName", msg, requestHeader, 1000 * 5, + CommunicationMode.SYNC, null); + System.out.println(i + "\t" + result); + } + catch (Exception e) { + e.printStackTrace(); + } + } + + client.shutdown(); + + brokerController.shutdown(); + } +} diff --git a/rocketmq-broker/src/test/java/com/alibaba/rocketmq/broker/offset/ConsumerOffsetManagerTest.java b/rocketmq-broker/src/test/java/com/alibaba/rocketmq/broker/offset/ConsumerOffsetManagerTest.java new file mode 100644 index 000000000..bb99d644c --- /dev/null +++ b/rocketmq-broker/src/test/java/com/alibaba/rocketmq/broker/offset/ConsumerOffsetManagerTest.java @@ -0,0 +1,52 @@ +/** + * $Id: ConsumerOffsetManagerTest.java 1831 2013-05-16 01:39:51Z shijia.wxr $ + */ +package com.alibaba.rocketmq.broker.offset; + +import java.util.Random; + +import org.junit.Test; + +import com.alibaba.rocketmq.broker.BrokerController; +import com.alibaba.rocketmq.common.BrokerConfig; +import com.alibaba.rocketmq.remoting.netty.NettyClientConfig; +import com.alibaba.rocketmq.remoting.netty.NettyServerConfig; +import com.alibaba.rocketmq.store.config.MessageStoreConfig; + + +/** + * @author shijia.wxr + */ +public class ConsumerOffsetManagerTest { + @Test + public void test_flushConsumerOffset() throws Exception { + BrokerController brokerController = new BrokerController(// + new BrokerConfig(), // + new NettyServerConfig(), // + new NettyClientConfig(), // + new MessageStoreConfig()); + boolean initResult = brokerController.initialize(); + System.out.println("initialize " + initResult); + brokerController.start(); + + ConsumerOffsetManager consumerOffsetManager = new ConsumerOffsetManager(brokerController); + + Random random = new Random(); + + for (int i = 0; i < 100; i++) { + String group = "DIANPU_GROUP_" + i; + for (int id = 0; id < 16; id++) { + consumerOffsetManager.commitOffset(group, "TOPIC_A", id, + random.nextLong() % 1024 * 1024 * 1024); + consumerOffsetManager.commitOffset(group, "TOPIC_B", id, + random.nextLong() % 1024 * 1024 * 1024); + consumerOffsetManager.commitOffset(group, "TOPIC_C", id, + random.nextLong() % 1024 * 1024 * 1024); + } + } + + consumerOffsetManager.persist(); + + brokerController.shutdown(); + } +} diff --git a/rocketmq-broker/src/test/java/com/alibaba/rocketmq/broker/topic/TopicConfigManagerTest.java b/rocketmq-broker/src/test/java/com/alibaba/rocketmq/broker/topic/TopicConfigManagerTest.java new file mode 100644 index 000000000..e67b0ed02 --- /dev/null +++ b/rocketmq-broker/src/test/java/com/alibaba/rocketmq/broker/topic/TopicConfigManagerTest.java @@ -0,0 +1,55 @@ +/** + * $Id: TopicConfigManagerTest.java 1831 2013-05-16 01:39:51Z shijia.wxr $ + */ +package com.alibaba.rocketmq.broker.topic; + +import static org.junit.Assert.assertTrue; + +import org.junit.Test; + +import com.alibaba.rocketmq.broker.BrokerController; +import com.alibaba.rocketmq.common.BrokerConfig; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.TopicConfig; +import com.alibaba.rocketmq.remoting.netty.NettyClientConfig; +import com.alibaba.rocketmq.remoting.netty.NettyServerConfig; +import com.alibaba.rocketmq.store.config.MessageStoreConfig; + + +/** + * @author shijia.wxr + */ +public class TopicConfigManagerTest { + @Test + public void test_flushTopicConfig() throws Exception { + BrokerController brokerController = new BrokerController(// + new BrokerConfig(), // + new NettyServerConfig(), // + new NettyClientConfig(), // + new MessageStoreConfig()); + boolean initResult = brokerController.initialize(); + System.out.println("initialize " + initResult); + brokerController.start(); + + TopicConfigManager topicConfigManager = new TopicConfigManager(brokerController); + + TopicConfig topicConfig = + topicConfigManager.createTopicInSendMessageMethod("TestTopic_SEND", MixAll.DEFAULT_TOPIC, + null, 4, 0); + assertTrue(topicConfig != null); + + System.out.println(topicConfig); + + for (int i = 0; i < 10; i++) { + String topic = "UNITTEST-" + i; + topicConfig = + topicConfigManager + .createTopicInSendMessageMethod(topic, MixAll.DEFAULT_TOPIC, null, 4, 0); + assertTrue(topicConfig != null); + } + + topicConfigManager.persist(); + + brokerController.shutdown(); + } +} diff --git a/rocketmq-broker/src/test/java/com/alibaba/rocketmq/broker/transaction/jdbc/JDBCTransactionStoreTest.java b/rocketmq-broker/src/test/java/com/alibaba/rocketmq/broker/transaction/jdbc/JDBCTransactionStoreTest.java new file mode 100644 index 000000000..56ac9e0bb --- /dev/null +++ b/rocketmq-broker/src/test/java/com/alibaba/rocketmq/broker/transaction/jdbc/JDBCTransactionStoreTest.java @@ -0,0 +1,94 @@ +package com.alibaba.rocketmq.broker.transaction.jdbc; + +import java.util.ArrayList; +import java.util.List; + +import org.junit.Assert; +import org.junit.Test; + +import com.alibaba.rocketmq.broker.transaction.TransactionRecord; +import com.alibaba.rocketmq.broker.transaction.TransactionStore; + + +public class JDBCTransactionStoreTest { + + @Test + public void test_derby_open() { + JDBCTransactionStoreConfig config = new JDBCTransactionStoreConfig(); + config.setJdbcDriverClass("org.apache.derby.jdbc.EmbeddedDriver"); + config.setJdbcURL("jdbc:derby:xxx;create=true"); + config.setJdbcUser("xxx"); + config.setJdbcPassword("xxx"); + TransactionStore store = new JDBCTransactionStore(config); + + boolean open = store.open(); + System.out.println(open); + Assert.assertTrue(open); + store.close(); + } + + + // @Test + public void test_mysql_open() { + JDBCTransactionStoreConfig config = new JDBCTransactionStoreConfig(); + + TransactionStore store = new JDBCTransactionStore(config); + + boolean open = store.open(); + System.out.println(open); + Assert.assertTrue(open); + store.close(); + } + + + // @Test + public void test_mysql_put() { + JDBCTransactionStoreConfig config = new JDBCTransactionStoreConfig(); + + TransactionStore store = new JDBCTransactionStore(config); + + boolean open = store.open(); + System.out.println(open); + Assert.assertTrue(open); + + long begin = System.currentTimeMillis(); + List trs = new ArrayList(); + for (int i = 0; i < 20; i++) { + TransactionRecord tr = new TransactionRecord(); + tr.setOffset(i); + tr.setProducerGroup("PG_" + i); + trs.add(tr); + } + + boolean write = store.put(trs); + + System.out.println("TIME=" + (System.currentTimeMillis() - begin)); + + Assert.assertTrue(write); + + store.close(); + } + + + // @Test + public void test_mysql_remove() { + JDBCTransactionStoreConfig config = new JDBCTransactionStoreConfig(); + + TransactionStore store = new JDBCTransactionStore(config); + + boolean open = store.open(); + System.out.println(open); + Assert.assertTrue(open); + + List pks = new ArrayList(); + pks.add(2L); + pks.add(4L); + pks.add(6L); + pks.add(8L); + pks.add(11L); + + store.remove(pks); + + store.close(); + } +} diff --git a/rocketmq-client/pom.xml b/rocketmq-client/pom.xml new file mode 100644 index 000000000..1f19212dd --- /dev/null +++ b/rocketmq-client/pom.xml @@ -0,0 +1,25 @@ + + + com.alibaba.rocketmq + rocketmq-all + 3.2.2 + + + 4.0.0 + jar + rocketmq-client + rocketmq-client ${project.version} + + + + junit + junit + test + + + ${project.groupId} + rocketmq-common + + + diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/ClientConfig.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/ClientConfig.java new file mode 100644 index 000000000..4ca21e153 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/ClientConfig.java @@ -0,0 +1,160 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client; + +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.remoting.common.RemotingUtil; + + +/** + * Producer与Consumer的公共配置 + * + * @author shijia.wxr + * @since 2013-7-24 + */ +public class ClientConfig { + private String namesrvAddr = System.getProperty(MixAll.NAMESRV_ADDR_PROPERTY, + System.getenv(MixAll.NAMESRV_ADDR_ENV)); + private String clientIP = RemotingUtil.getLocalAddress(); + private String instanceName = System.getProperty("rocketmq.client.name", "DEFAULT"); + private int clientCallbackExecutorThreads = Runtime.getRuntime().availableProcessors(); + private int pollNameServerInteval = 1000 * 30; + private int heartbeatBrokerInterval = 1000 * 30; + private int persistConsumerOffsetInterval = 1000 * 5; + + + public String buildMQClientId() { + StringBuilder sb = new StringBuilder(); + sb.append(this.getClientIP()); + + sb.append("@"); + sb.append(this.getInstanceName()); + + return sb.toString(); + } + + + public void changeInstanceNameToPID() { + if (this.instanceName.equals("DEFAULT")) { + this.instanceName = String.valueOf(UtilAll.getPid()); + } + } + + + public void resetClientConfig(final ClientConfig cc) { + this.namesrvAddr = cc.namesrvAddr; + this.clientIP = cc.clientIP; + this.instanceName = cc.instanceName; + this.clientCallbackExecutorThreads = cc.clientCallbackExecutorThreads; + this.pollNameServerInteval = cc.pollNameServerInteval; + this.heartbeatBrokerInterval = cc.heartbeatBrokerInterval; + this.persistConsumerOffsetInterval = cc.persistConsumerOffsetInterval; + } + + + public ClientConfig cloneClientConfig() { + ClientConfig cc = new ClientConfig(); + cc.namesrvAddr = namesrvAddr; + cc.clientIP = clientIP; + cc.instanceName = instanceName; + cc.clientCallbackExecutorThreads = clientCallbackExecutorThreads; + cc.pollNameServerInteval = pollNameServerInteval; + cc.heartbeatBrokerInterval = heartbeatBrokerInterval; + cc.persistConsumerOffsetInterval = persistConsumerOffsetInterval; + return cc; + } + + + public String getNamesrvAddr() { + return namesrvAddr; + } + + + public void setNamesrvAddr(String namesrvAddr) { + this.namesrvAddr = namesrvAddr; + } + + + public String getClientIP() { + return clientIP; + } + + + public void setClientIP(String clientIP) { + this.clientIP = clientIP; + } + + + public String getInstanceName() { + return instanceName; + } + + + public void setInstanceName(String instanceName) { + this.instanceName = instanceName; + } + + + public int getClientCallbackExecutorThreads() { + return clientCallbackExecutorThreads; + } + + + public void setClientCallbackExecutorThreads(int clientCallbackExecutorThreads) { + this.clientCallbackExecutorThreads = clientCallbackExecutorThreads; + } + + + public int getPollNameServerInteval() { + return pollNameServerInteval; + } + + + public void setPollNameServerInteval(int pollNameServerInteval) { + this.pollNameServerInteval = pollNameServerInteval; + } + + + public int getHeartbeatBrokerInterval() { + return heartbeatBrokerInterval; + } + + + public void setHeartbeatBrokerInterval(int heartbeatBrokerInterval) { + this.heartbeatBrokerInterval = heartbeatBrokerInterval; + } + + + public int getPersistConsumerOffsetInterval() { + return persistConsumerOffsetInterval; + } + + + public void setPersistConsumerOffsetInterval(int persistConsumerOffsetInterval) { + this.persistConsumerOffsetInterval = persistConsumerOffsetInterval; + } + + + @Override + public String toString() { + return "ClientConfig [namesrvAddr=" + namesrvAddr + ", clientIP=" + clientIP + ", instanceName=" + + instanceName + ", clientCallbackExecutorThreads=" + clientCallbackExecutorThreads + + ", pollNameServerInteval=" + pollNameServerInteval + ", heartbeatBrokerInterval=" + + heartbeatBrokerInterval + ", persistConsumerOffsetInterval=" + + persistConsumerOffsetInterval + "]"; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/MQAdmin.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/MQAdmin.java new file mode 100644 index 000000000..c853a74ad --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/MQAdmin.java @@ -0,0 +1,144 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client; + +import com.alibaba.rocketmq.client.exception.MQBrokerException; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.common.message.MessageExt; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.remoting.exception.RemotingException; + + +/** + * MQ管理类接口 + * + * @author shijia.wxr + * @since 2013-7-24 + */ +public interface MQAdmin { + /** + * 创建topic + * + * @param key + * 请向运维人员申请 + * @param newTopic + * 要创建的新topic + * @param queueNum + * 新topic队列数 + * @throws MQClientException + */ + public void createTopic(final String key, final String newTopic, final int queueNum) + throws MQClientException; + + + /** + * 创建topic + * + * @param key + * 请向运维人员申请 + * @param newTopic + * 要创建的新topic + * @param queueNum + * 新topic队列数 + * @param topicSysFlag + * 新 topic 配置标识 + * @throws MQClientException + */ + public void createTopic(String key, String newTopic, int queueNum, int topicSysFlag) + throws MQClientException; + + + /** + * 根据时间查询对应的offset，精确到毫秒
+ * P.S. 当前接口有较多IO开销，请勿频繁调用 + * + * @param mq + * 队列 + * @param timestamp + * 毫秒形式时间戳 + * @return 指定时间对应的offset + * @throws MQClientException + */ + public long searchOffset(final MessageQueue mq, final long timestamp) throws MQClientException; + + + /** + * 向服务器查询队列最大Offset PS: 最大Offset无对应消息，减1有消息 + * + * @param mq + * 队列 + * @return 队列的最大Offset + * @throws MQClientException + */ + public long maxOffset(final MessageQueue mq) throws MQClientException; + + + /** + * 向服务器查询队列最小Offset PS: 最小Offset有对应消息 + * + * @param mq + * 队列 + * @return 队列的最小Offset + * @throws MQClientException + */ + public long minOffset(final MessageQueue mq) throws MQClientException; + + + /** + * 向服务器查询队列保存的最早消息对应的存储时间 + * + * @param mq + * 队列 + * @return 最早消息对应的存储时间，精确到毫秒 + * @throws MQClientException + */ + public long earliestMsgStoreTime(final MessageQueue mq) throws MQClientException; + + + /** + * 根据消息ID，从服务器获取完整的消息 + * + * @param msgId + * @return 完整消息 + * @throws InterruptedException + * @throws MQBrokerException + * @throws RemotingException + * @throws MQClientException + */ + public MessageExt viewMessage(final String msgId) throws RemotingException, MQBrokerException, + InterruptedException, MQClientException; + + + /** + * 根据消息Key查询消息 + * + * @param topic + * 消息主题 + * @param key + * 消息关键词 + * @param maxNum + * 查询最大条数 + * @param begin + * 起始时间戳 + * @param end + * 结束时间戳 + * @return 查询结果 + * @throws MQClientException + * @throws InterruptedException + */ + public QueryResult queryMessage(final String topic, final String key, final int maxNum, final long begin, + final long end) throws MQClientException, InterruptedException; +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/MQHelper.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/MQHelper.java new file mode 100644 index 000000000..f69b6ba18 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/MQHelper.java @@ -0,0 +1,97 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client; + +import java.util.Set; +import java.util.TreeSet; + +import org.slf4j.Logger; + +import com.alibaba.rocketmq.client.consumer.DefaultMQPullConsumer; +import com.alibaba.rocketmq.client.log.ClientLogger; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.common.protocol.heartbeat.MessageModel; + + +/** + * @author shijia.wxr + * @since 2013-11-13 + */ +public class MQHelper { + /** + * 根据时间戳来重置一个订阅组的消费进度 + * + * @param messageModel + * 广播消费还是集群消费 + * @param instanceName + * 实例名称，保持与工作Consumer一致。 + * @param consumerGroup + * 订阅组 + * @param topic + * topic + * @param timestamp + * 时间戳 + * @throws Exception + */ + public static void resetOffsetByTimestamp(// + final MessageModel messageModel,// + final String instanceName,// + final String consumerGroup, // + final String topic, // + final long timestamp) throws Exception { + final Logger log = ClientLogger.getLog(); + + DefaultMQPullConsumer consumer = new DefaultMQPullConsumer(consumerGroup); + consumer.setInstanceName(instanceName); + consumer.setMessageModel(messageModel); + consumer.start(); + + Set mqs = null; + try { + mqs = consumer.fetchSubscribeMessageQueues(topic); + if (mqs != null && !mqs.isEmpty()) { + TreeSet mqsNew = new TreeSet(mqs); + for (MessageQueue mq : mqsNew) { + long offset = consumer.searchOffset(mq, timestamp); + if (offset >= 0) { + consumer.updateConsumeOffset(mq, offset); + log.info("resetOffsetByTimestamp updateConsumeOffset success, {} {} {}", + consumerGroup, offset, mq); + } + } + } + } + catch (Exception e) { + log.warn("resetOffsetByTimestamp Exception", e); + throw e; + } + finally { + if (mqs != null) { + consumer.getDefaultMQPullConsumerImpl().getOffsetStore().persistAll(mqs); + } + consumer.shutdown(); + } + } + + + public static void resetOffsetByTimestamp(// + final MessageModel messageModel,// + final String consumerGroup, // + final String topic, // + final long timestamp) throws Exception { + resetOffsetByTimestamp(messageModel, "DEFAULT", consumerGroup, topic, timestamp); + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/QueryResult.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/QueryResult.java new file mode 100644 index 000000000..0ffab4a24 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/QueryResult.java @@ -0,0 +1,55 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client; + +import java.util.List; + +import com.alibaba.rocketmq.common.message.MessageExt; + + +/** + * 查询消息返回结果 + * + * @author shijia.wxr + * @since 2013-7-24 + */ +public class QueryResult { + private final long indexLastUpdateTimestamp; + private final List messageList; + + + public QueryResult(long indexLastUpdateTimestamp, List messageList) { + this.indexLastUpdateTimestamp = indexLastUpdateTimestamp; + this.messageList = messageList; + } + + + public long getIndexLastUpdateTimestamp() { + return indexLastUpdateTimestamp; + } + + + public List getMessageList() { + return messageList; + } + + + @Override + public String toString() { + return "QueryResult [indexLastUpdateTimestamp=" + indexLastUpdateTimestamp + ", messageList=" + + messageList + "]"; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/Validators.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/Validators.java new file mode 100644 index 000000000..7c7823d85 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/Validators.java @@ -0,0 +1,137 @@ +package com.alibaba.rocketmq.client; + +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.client.producer.DefaultMQProducer; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.message.Message; +import com.alibaba.rocketmq.common.protocol.ResponseCode; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + + +/** + * 有效性检查公用类。 + * + * @author manhong.yqd + * @since 2013-8-28 + */ +public class Validators { + public static final String VALID_PATTERN_STR = "^[%|a-zA-Z0-9_-]+$"; + public static final Pattern PATTERN = Pattern.compile(VALID_PATTERN_STR); + public static final int CHARACTER_MAX_LENGTH = 255; + + + /** + * 通过正则表达式进行字符匹配 + * + * @param origin + * @param pattern + * @return + */ + public static boolean regularExpressionMatcher(String origin, Pattern pattern) { + if (pattern == null) { + return true; + } + Matcher matcher = pattern.matcher(origin); + return matcher.matches(); + } + + + /** + * 通过正则表达式查找匹配的字符 + * + * @param origin + * @param patternStr + * @return + */ + public static String getGroupWithRegularExpression(String origin, String patternStr) { + Pattern pattern = Pattern.compile(patternStr); + Matcher matcher = pattern.matcher(origin); + while (matcher.find()) { + return matcher.group(0); + } + return null; + } + + + /** + * topic 有效性检查 + * + * @param topic + * @throws com.alibaba.rocketmq.client.exception.MQClientException + */ + public static void checkTopic(String topic) throws MQClientException { + if (UtilAll.isBlank(topic)) { + throw new MQClientException("the specified topic is blank", null); + } + + if (!regularExpressionMatcher(topic, PATTERN)) { + throw new MQClientException(String.format( + "the specified topic[%s] contains illegal characters, allowing only %s", topic, + VALID_PATTERN_STR), null); + } + + if (topic.length() > CHARACTER_MAX_LENGTH) { + throw new MQClientException("the specified topic is longer than topic max length 255.", null); + } + + // Topic名字是否与保留字段冲突 + if (topic.equals(MixAll.DEFAULT_TOPIC)) { + throw new MQClientException( + String.format("the topic[%s] is conflict with default topic.", topic), null); + } + } + + + /** + * group 有效性检查 + * + * @param group + * @throws com.alibaba.rocketmq.client.exception.MQClientException + */ + public static void checkGroup(String group) throws MQClientException { + if (UtilAll.isBlank(group)) { + throw new MQClientException("the specified group is blank", null); + } + if (!regularExpressionMatcher(group, PATTERN)) { + throw new MQClientException(String.format( + "the specified group[%s] contains illegal characters, allowing only %s", group, + VALID_PATTERN_STR), null); + } + if (group.length() > CHARACTER_MAX_LENGTH) { + throw new MQClientException("the specified group is longer than group max length 255.", null); + } + } + + + /** + * message 有效性检查 + * + * @param msg + * @param defaultMQProducer + * @throws com.alibaba.rocketmq.client.exception.MQClientException + */ + public static void checkMessage(Message msg, DefaultMQProducer defaultMQProducer) + throws MQClientException { + if (null == msg) { + throw new MQClientException(ResponseCode.MESSAGE_ILLEGAL, "the message is null"); + } + // topic + Validators.checkTopic(msg.getTopic()); + // body + if (null == msg.getBody()) { + throw new MQClientException(ResponseCode.MESSAGE_ILLEGAL, "the message body is null"); + } + + if (0 == msg.getBody().length) { + throw new MQClientException(ResponseCode.MESSAGE_ILLEGAL, "the message body length is zero"); + } + + if (msg.getBody().length > defaultMQProducer.getMaxMessageSize()) { + throw new MQClientException(ResponseCode.MESSAGE_ILLEGAL, + "the message body size over max value, MAX: " + defaultMQProducer.getMaxMessageSize()); + } + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/VirtualEnvUtil.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/VirtualEnvUtil.java new file mode 100644 index 000000000..8ac55e720 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/VirtualEnvUtil.java @@ -0,0 +1,63 @@ +package com.alibaba.rocketmq.client; + +import com.alibaba.rocketmq.common.UtilAll; + + +/** + * 虚拟环境相关 API 封装 + * + * @author manhong.yqd + * @since 2013-8-26 + */ +public class VirtualEnvUtil { + public static final String VIRTUAL_APPGROUP_PREFIX = "%%PROJECT_%s%%"; + + + /** + * 添加虚拟运行环境相关的projectGroupPrefix + * + * @param origin + * @param projectGroup + * @return + */ + public static String buildWithProjectGroup(String origin, String projectGroup) { + if (!UtilAll.isBlank(projectGroup)) { + String prefix = String.format(VIRTUAL_APPGROUP_PREFIX, projectGroup); + if (!origin.endsWith(prefix)) { + return origin + prefix; + } + else { + return origin; + } + } + else { + return origin; + } + } + + + /** + * 清除虚拟运行环境相关的projectGroupPrefix + * + * @param origin + * @param projectGroup + * @return + */ + public static String clearProjectGroup(String origin, String projectGroup) { + String prefix = String.format(VIRTUAL_APPGROUP_PREFIX, projectGroup); + if (!UtilAll.isBlank(prefix) && origin.endsWith(prefix)) { + return origin.substring(0, origin.lastIndexOf(prefix)); + } + else { + return origin; + } + } + + + public static void main(String[] args) { + String ori = "bbbb"; + String str = buildWithProjectGroup(ori, "AAA"); + System.out.println("build=" + str); + System.out.println("ori=" + clearProjectGroup(str, "AAA")); + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/admin/MQAdminExtInner.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/admin/MQAdminExtInner.java new file mode 100644 index 000000000..d6b9dd349 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/admin/MQAdminExtInner.java @@ -0,0 +1,24 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.admin; + +/** + * @author shijia.wxr + * @since 2013-7-14 + */ +public interface MQAdminExtInner { + +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/AllocateMessageQueueStrategy.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/AllocateMessageQueueStrategy.java new file mode 100644 index 000000000..8a476299a --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/AllocateMessageQueueStrategy.java @@ -0,0 +1,55 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.consumer; + +import java.util.List; + +import com.alibaba.rocketmq.common.message.MessageQueue; + + +/** + * Consumer队列自动分配策略 + * + * @author shijia.wxr + * @since 2013-7-24 + */ +public interface AllocateMessageQueueStrategy { + /** + * 给当前的ConsumerId分配队列 + * + * @param currentCID + * 当前ConsumerId + * @param mqAll + * 当前Topic的所有队列集合，无重复数据，且有序 + * @param cidAll + * 当前订阅组的所有Consumer集合，无重复数据，且有序 + * @return 分配结果，无重复数据 + */ + public List allocate(// + final String consumerGroup,// + final String currentCID,// + final List mqAll,// + final List cidAll// + ); + + + /** + * rebalance 算法的名字 + * + * @return + */ + public String getName(); +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/DefaultMQPullConsumer.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/DefaultMQPullConsumer.java new file mode 100644 index 000000000..073c147c0 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/DefaultMQPullConsumer.java @@ -0,0 +1,352 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.consumer; + +import java.util.HashSet; +import java.util.Set; + +import com.alibaba.rocketmq.client.ClientConfig; +import com.alibaba.rocketmq.client.QueryResult; +import com.alibaba.rocketmq.client.consumer.rebalance.AllocateMessageQueueAveragely; +import com.alibaba.rocketmq.client.consumer.store.OffsetStore; +import com.alibaba.rocketmq.client.exception.MQBrokerException; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.client.impl.consumer.DefaultMQPullConsumerImpl; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.message.MessageExt; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.common.protocol.heartbeat.MessageModel; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.remoting.exception.RemotingException; + + +/** + * 消费者，主动拉取方式消费 + * + * @author shijia.wxr + * @since 2013-7-24 + */ +public class DefaultMQPullConsumer extends ClientConfig implements MQPullConsumer { + protected final transient DefaultMQPullConsumerImpl defaultMQPullConsumerImpl; + + /** + * 做同样事情的Consumer归为同一个Group，应用必须设置，并保证命名唯一 + */ + private String consumerGroup; + /** + * 长轮询模式，Consumer连接在Broker挂起最长时间，不建议修改 + */ + private long brokerSuspendMaxTimeMillis = 1000 * 20; + /** + * 长轮询模式，Consumer超时时间（必须要大于brokerSuspendMaxTimeMillis），不建议修改 + */ + private long consumerTimeoutMillisWhenSuspend = 1000 * 30; + /** + * 非阻塞拉模式，Consumer超时时间，不建议修改 + */ + private long consumerPullTimeoutMillis = 1000 * 10; + /** + * 集群消费/广播消费 + */ + private MessageModel messageModel = MessageModel.CLUSTERING; + /** + * 队列变化监听器 + */ + private MessageQueueListener messageQueueListener; + /** + * Offset存储，系统会根据客户端配置自动创建相应的实现，如果应用配置了，则以应用配置的为主 + */ + private OffsetStore offsetStore; + /** + * 需要监听哪些Topic的队列变化 + */ + private Set registerTopics = new HashSet(); + /** + * 队列分配算法，应用可重写 + */ + private AllocateMessageQueueStrategy allocateMessageQueueStrategy = new AllocateMessageQueueAveragely(); + /** + * 是否为单元化的订阅组 + */ + private boolean unitMode = false; + + + public DefaultMQPullConsumer() { + this(MixAll.DEFAULT_CONSUMER_GROUP, null); + } + + + public DefaultMQPullConsumer(final String consumerGroup) { + this(consumerGroup, null); + } + + + public DefaultMQPullConsumer(RPCHook rpcHook) { + this(MixAll.DEFAULT_CONSUMER_GROUP, rpcHook); + } + + + public DefaultMQPullConsumer(final String consumerGroup, RPCHook rpcHook) { + this.consumerGroup = consumerGroup; + defaultMQPullConsumerImpl = new DefaultMQPullConsumerImpl(this, rpcHook); + } + + + @Override + public void createTopic(String key, String newTopic, int queueNum) throws MQClientException { + createTopic(key, newTopic, queueNum, 0); + } + + + @Override + public void createTopic(String key, String newTopic, int queueNum, int topicSysFlag) + throws MQClientException { + this.defaultMQPullConsumerImpl.createTopic(key, newTopic, queueNum, topicSysFlag); + } + + + @Override + public long searchOffset(MessageQueue mq, long timestamp) throws MQClientException { + return this.defaultMQPullConsumerImpl.searchOffset(mq, timestamp); + } + + + @Override + public long maxOffset(MessageQueue mq) throws MQClientException { + return this.defaultMQPullConsumerImpl.maxOffset(mq); + } + + + @Override + public long minOffset(MessageQueue mq) throws MQClientException { + return this.defaultMQPullConsumerImpl.minOffset(mq); + } + + + @Override + public long earliestMsgStoreTime(MessageQueue mq) throws MQClientException { + return this.defaultMQPullConsumerImpl.earliestMsgStoreTime(mq); + } + + + @Override + public MessageExt viewMessage(String msgId) throws RemotingException, MQBrokerException, + InterruptedException, MQClientException { + return this.defaultMQPullConsumerImpl.viewMessage(msgId); + } + + + @Override + public QueryResult queryMessage(String topic, String key, int maxNum, long begin, long end) + throws MQClientException, InterruptedException { + return this.defaultMQPullConsumerImpl.queryMessage(topic, key, maxNum, begin, end); + } + + + public AllocateMessageQueueStrategy getAllocateMessageQueueStrategy() { + return allocateMessageQueueStrategy; + } + + + public void setAllocateMessageQueueStrategy(AllocateMessageQueueStrategy allocateMessageQueueStrategy) { + this.allocateMessageQueueStrategy = allocateMessageQueueStrategy; + } + + + public long getBrokerSuspendMaxTimeMillis() { + return brokerSuspendMaxTimeMillis; + } + + + public void setBrokerSuspendMaxTimeMillis(long brokerSuspendMaxTimeMillis) { + this.brokerSuspendMaxTimeMillis = brokerSuspendMaxTimeMillis; + } + + + public String getConsumerGroup() { + return consumerGroup; + } + + + public void setConsumerGroup(String consumerGroup) { + this.consumerGroup = consumerGroup; + } + + + public long getConsumerPullTimeoutMillis() { + return consumerPullTimeoutMillis; + } + + + public void setConsumerPullTimeoutMillis(long consumerPullTimeoutMillis) { + this.consumerPullTimeoutMillis = consumerPullTimeoutMillis; + } + + + public long getConsumerTimeoutMillisWhenSuspend() { + return consumerTimeoutMillisWhenSuspend; + } + + + public void setConsumerTimeoutMillisWhenSuspend(long consumerTimeoutMillisWhenSuspend) { + this.consumerTimeoutMillisWhenSuspend = consumerTimeoutMillisWhenSuspend; + } + + + public MessageModel getMessageModel() { + return messageModel; + } + + + public void setMessageModel(MessageModel messageModel) { + this.messageModel = messageModel; + } + + + public MessageQueueListener getMessageQueueListener() { + return messageQueueListener; + } + + + public void setMessageQueueListener(MessageQueueListener messageQueueListener) { + this.messageQueueListener = messageQueueListener; + } + + + public Set getRegisterTopics() { + return registerTopics; + } + + + public void setRegisterTopics(Set registerTopics) { + this.registerTopics = registerTopics; + } + + + @Override + public void sendMessageBack(MessageExt msg, int delayLevel) throws RemotingException, MQBrokerException, + InterruptedException, MQClientException { + this.defaultMQPullConsumerImpl.sendMessageBack(msg, delayLevel, null); + } + + + @Override + public void sendMessageBack(MessageExt msg, int delayLevel, String brokerName) throws RemotingException, + MQBrokerException, InterruptedException, MQClientException { + this.defaultMQPullConsumerImpl.sendMessageBack(msg, delayLevel, brokerName); + } + + + @Override + public Set fetchSubscribeMessageQueues(String topic) throws MQClientException { + return this.defaultMQPullConsumerImpl.fetchSubscribeMessageQueues(topic); + } + + + @Override + public void start() throws MQClientException { + this.defaultMQPullConsumerImpl.start(); + } + + + @Override + public void shutdown() { + this.defaultMQPullConsumerImpl.shutdown(); + } + + + @Override + public void registerMessageQueueListener(String topic, MessageQueueListener listener) { + synchronized (this.registerTopics) { + this.registerTopics.add(topic); + if (listener != null) { + this.messageQueueListener = listener; + } + } + } + + + @Override + public PullResult pull(MessageQueue mq, String subExpression, long offset, int maxNums) + throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + return this.defaultMQPullConsumerImpl.pull(mq, subExpression, offset, maxNums); + } + + + @Override + public void pull(MessageQueue mq, String subExpression, long offset, int maxNums, + PullCallback pullCallback) throws MQClientException, RemotingException, InterruptedException { + this.defaultMQPullConsumerImpl.pull(mq, subExpression, offset, maxNums, pullCallback); + } + + + @Override + public PullResult pullBlockIfNotFound(MessageQueue mq, String subExpression, long offset, int maxNums) + throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + return this.defaultMQPullConsumerImpl.pullBlockIfNotFound(mq, subExpression, offset, maxNums); + } + + + @Override + public void pullBlockIfNotFound(MessageQueue mq, String subExpression, long offset, int maxNums, + PullCallback pullCallback) throws MQClientException, RemotingException, InterruptedException { + this.defaultMQPullConsumerImpl.pullBlockIfNotFound(mq, subExpression, offset, maxNums, pullCallback); + } + + + @Override + public void updateConsumeOffset(MessageQueue mq, long offset) throws MQClientException { + this.defaultMQPullConsumerImpl.updateConsumeOffset(mq, offset); + } + + + @Override + public long fetchConsumeOffset(MessageQueue mq, boolean fromStore) throws MQClientException { + return this.defaultMQPullConsumerImpl.fetchConsumeOffset(mq, fromStore); + } + + + @Override + public Set fetchMessageQueuesInBalance(String topic) throws MQClientException { + return this.defaultMQPullConsumerImpl.fetchMessageQueuesInBalance(topic); + } + + + public OffsetStore getOffsetStore() { + return offsetStore; + } + + + public void setOffsetStore(OffsetStore offsetStore) { + this.offsetStore = offsetStore; + } + + + public DefaultMQPullConsumerImpl getDefaultMQPullConsumerImpl() { + return defaultMQPullConsumerImpl; + } + + + public boolean isUnitMode() { + return unitMode; + } + + + public void setUnitMode(boolean isUnitMode) { + this.unitMode = isUnitMode; + } + +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/DefaultMQPushConsumer.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/DefaultMQPushConsumer.java new file mode 100644 index 000000000..38c9fb493 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/DefaultMQPushConsumer.java @@ -0,0 +1,459 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.consumer; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +import com.alibaba.rocketmq.client.ClientConfig; +import com.alibaba.rocketmq.client.QueryResult; +import com.alibaba.rocketmq.client.consumer.listener.MessageListener; +import com.alibaba.rocketmq.client.consumer.rebalance.AllocateMessageQueueAveragely; +import com.alibaba.rocketmq.client.consumer.store.OffsetStore; +import com.alibaba.rocketmq.client.exception.MQBrokerException; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.client.impl.consumer.DefaultMQPushConsumerImpl; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.consumer.ConsumeFromWhere; +import com.alibaba.rocketmq.common.message.MessageExt; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.common.protocol.heartbeat.MessageModel; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.remoting.exception.RemotingException; + + +/** + * 类似于Broker Push消息到Consumer方式，但实际仍然是Consumer内部后台从Broker Pull消息
+ * 采用长轮询方式拉消息，实时性同push方式一致，且不会无谓的拉消息导致Broker、Consumer压力增大 + * + * @author shijia.wxr + * @since 2013-7-24 + */ +public class DefaultMQPushConsumer extends ClientConfig implements MQPushConsumer { + protected final transient DefaultMQPushConsumerImpl defaultMQPushConsumerImpl; + /** + * 做同样事情的Consumer归为同一个Group，应用必须设置，并保证命名唯一 + */ + private String consumerGroup; + /** + * 集群消费/广播消费 + */ + private MessageModel messageModel = MessageModel.CLUSTERING; + /** + * Consumer第一次启动时，从哪里开始消费 + */ + private ConsumeFromWhere consumeFromWhere = ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET; + /** + * Consumer第一次启动时，如果回溯消费，默认回溯到哪个时间点，数据格式如下，时间精度秒：
+ * 20131223171201
+ * 表示2013年12月23日17点12分01秒
+ * 默认回溯到相对启动时间的半小时前 + */ + private String consumeTimestamp = UtilAll.timeMillisToHumanString3(System.currentTimeMillis() + - (1000 * 60 * 30)); + /** + * 队列分配算法，应用可重写 + */ + private AllocateMessageQueueStrategy allocateMessageQueueStrategy; + + /** + * 订阅关系 + */ + private Map subscription = new HashMap(); + /** + * 消息监听器 + */ + private MessageListener messageListener; + /** + * Offset存储，系统会根据客户端配置自动创建相应的实现，如果应用配置了，则以应用配置的为主 + */ + private OffsetStore offsetStore; + /** + * 消费消息线程，最小数目 + */ + private int consumeThreadMin = 20; + /** + * 消费消息线程，最大数目 + */ + private int consumeThreadMax = 64; + + /** + * 消息堆积超过此阀值，动态调整线程池数 + */ + private long adjustThreadPoolNumsThreshold = 100000; + + /** + * 同一队列并行消费的最大跨度，顺序消费方式情况下，此参数无效 + */ + private int consumeConcurrentlyMaxSpan = 2000; + /** + * 本地队列消息数超过此阀值，开始流控 + */ + private int pullThresholdForQueue = 1000; + /** + * 拉消息间隔，如果为了降低拉取速度，可以设置大于0的值 + */ + private long pullInterval = 0; + /** + * 消费一批消息，最大数 + */ + private int consumeMessageBatchMaxSize = 1; + /** + * 拉消息，一次拉多少条 + */ + private int pullBatchSize = 32; + + /** + * 是否每次拉消息时，都上传订阅关系 + */ + private boolean postSubscriptionWhenPull = false; + + /** + * 是否为单元化的订阅组 + */ + private boolean unitMode = false; + + + public DefaultMQPushConsumer() { + this(MixAll.DEFAULT_CONSUMER_GROUP, null, new AllocateMessageQueueAveragely()); + } + + + public DefaultMQPushConsumer(RPCHook rpcHook) { + this(MixAll.DEFAULT_CONSUMER_GROUP, rpcHook, new AllocateMessageQueueAveragely()); + } + + + public DefaultMQPushConsumer(final String consumerGroup) { + this(consumerGroup, null, new AllocateMessageQueueAveragely()); + } + + + public DefaultMQPushConsumer(final String consumerGroup, RPCHook rpcHook, + AllocateMessageQueueStrategy allocateMessageQueueStrategy) { + this.consumerGroup = consumerGroup; + this.allocateMessageQueueStrategy = allocateMessageQueueStrategy; + defaultMQPushConsumerImpl = new DefaultMQPushConsumerImpl(this, rpcHook); + } + + + @Override + public void createTopic(String key, String newTopic, int queueNum) throws MQClientException { + createTopic(key, newTopic, queueNum, 0); + } + + + @Override + public void createTopic(String key, String newTopic, int queueNum, int topicSysFlag) + throws MQClientException { + this.defaultMQPushConsumerImpl.createTopic(key, newTopic, queueNum, topicSysFlag); + } + + + @Override + public long searchOffset(MessageQueue mq, long timestamp) throws MQClientException { + return this.defaultMQPushConsumerImpl.searchOffset(mq, timestamp); + } + + + @Override + public long maxOffset(MessageQueue mq) throws MQClientException { + return this.defaultMQPushConsumerImpl.maxOffset(mq); + } + + + @Override + public long minOffset(MessageQueue mq) throws MQClientException { + return this.defaultMQPushConsumerImpl.minOffset(mq); + } + + + @Override + public long earliestMsgStoreTime(MessageQueue mq) throws MQClientException { + return this.defaultMQPushConsumerImpl.earliestMsgStoreTime(mq); + } + + + @Override + public MessageExt viewMessage(String msgId) throws RemotingException, MQBrokerException, + InterruptedException, MQClientException { + return this.defaultMQPushConsumerImpl.viewMessage(msgId); + } + + + @Override + public QueryResult queryMessage(String topic, String key, int maxNum, long begin, long end) + throws MQClientException, InterruptedException { + return this.defaultMQPushConsumerImpl.queryMessage(topic, key, maxNum, begin, end); + } + + + public AllocateMessageQueueStrategy getAllocateMessageQueueStrategy() { + return allocateMessageQueueStrategy; + } + + + public void setAllocateMessageQueueStrategy(AllocateMessageQueueStrategy allocateMessageQueueStrategy) { + this.allocateMessageQueueStrategy = allocateMessageQueueStrategy; + } + + + public int getConsumeConcurrentlyMaxSpan() { + return consumeConcurrentlyMaxSpan; + } + + + public void setConsumeConcurrentlyMaxSpan(int consumeConcurrentlyMaxSpan) { + this.consumeConcurrentlyMaxSpan = consumeConcurrentlyMaxSpan; + } + + + public ConsumeFromWhere getConsumeFromWhere() { + return consumeFromWhere; + } + + + public void setConsumeFromWhere(ConsumeFromWhere consumeFromWhere) { + this.consumeFromWhere = consumeFromWhere; + } + + + public int getConsumeMessageBatchMaxSize() { + return consumeMessageBatchMaxSize; + } + + + public void setConsumeMessageBatchMaxSize(int consumeMessageBatchMaxSize) { + this.consumeMessageBatchMaxSize = consumeMessageBatchMaxSize; + } + + + public String getConsumerGroup() { + return consumerGroup; + } + + + public void setConsumerGroup(String consumerGroup) { + this.consumerGroup = consumerGroup; + } + + + public int getConsumeThreadMax() { + return consumeThreadMax; + } + + + public void setConsumeThreadMax(int consumeThreadMax) { + this.consumeThreadMax = consumeThreadMax; + } + + + public int getConsumeThreadMin() { + return consumeThreadMin; + } + + + public void setConsumeThreadMin(int consumeThreadMin) { + this.consumeThreadMin = consumeThreadMin; + } + + + public DefaultMQPushConsumerImpl getDefaultMQPushConsumerImpl() { + return defaultMQPushConsumerImpl; + } + + + public MessageListener getMessageListener() { + return messageListener; + } + + + public void setMessageListener(MessageListener messageListener) { + this.messageListener = messageListener; + } + + + public MessageModel getMessageModel() { + return messageModel; + } + + + public void setMessageModel(MessageModel messageModel) { + this.messageModel = messageModel; + } + + + public int getPullBatchSize() { + return pullBatchSize; + } + + + public void setPullBatchSize(int pullBatchSize) { + this.pullBatchSize = pullBatchSize; + } + + + public long getPullInterval() { + return pullInterval; + } + + + public void setPullInterval(long pullInterval) { + this.pullInterval = pullInterval; + } + + + public int getPullThresholdForQueue() { + return pullThresholdForQueue; + } + + + public void setPullThresholdForQueue(int pullThresholdForQueue) { + this.pullThresholdForQueue = pullThresholdForQueue; + } + + + public Map getSubscription() { + return subscription; + } + + + public void setSubscription(Map subscription) { + this.subscription = subscription; + } + + + @Override + public void sendMessageBack(MessageExt msg, int delayLevel) throws RemotingException, MQBrokerException, + InterruptedException, MQClientException { + this.defaultMQPushConsumerImpl.sendMessageBack(msg, delayLevel, null); + } + + + @Override + public void sendMessageBack(MessageExt msg, int delayLevel, String brokerName) throws RemotingException, + MQBrokerException, InterruptedException, MQClientException { + this.defaultMQPushConsumerImpl.sendMessageBack(msg, delayLevel, brokerName); + } + + + @Override + public Set fetchSubscribeMessageQueues(String topic) throws MQClientException { + return this.defaultMQPushConsumerImpl.fetchSubscribeMessageQueues(topic); + } + + + @Override + public void start() throws MQClientException { + this.defaultMQPushConsumerImpl.start(); + } + + + @Override + public void shutdown() { + this.defaultMQPushConsumerImpl.shutdown(); + } + + + @Override + public void registerMessageListener(MessageListener messageListener) { + this.messageListener = messageListener; + this.defaultMQPushConsumerImpl.registerMessageListener(messageListener); + } + + + @Override + public void subscribe(String topic, String subExpression) throws MQClientException { + this.defaultMQPushConsumerImpl.subscribe(topic, subExpression); + } + + + @Override + public void unsubscribe(String topic) { + this.defaultMQPushConsumerImpl.unsubscribe(topic); + } + + + @Override + public void updateCorePoolSize(int corePoolSize) { + this.defaultMQPushConsumerImpl.updateCorePoolSize(corePoolSize); + } + + + @Override + public void suspend() { + this.defaultMQPushConsumerImpl.suspend(); + } + + + @Override + public void resume() { + this.defaultMQPushConsumerImpl.resume(); + } + + + public OffsetStore getOffsetStore() { + return offsetStore; + } + + + public void setOffsetStore(OffsetStore offsetStore) { + this.offsetStore = offsetStore; + } + + + public String getConsumeTimestamp() { + return consumeTimestamp; + } + + + public void setConsumeTimestamp(String consumeTimestamp) { + this.consumeTimestamp = consumeTimestamp; + } + + + public boolean isPostSubscriptionWhenPull() { + return postSubscriptionWhenPull; + } + + + public void setPostSubscriptionWhenPull(boolean postSubscriptionWhenPull) { + this.postSubscriptionWhenPull = postSubscriptionWhenPull; + } + + + public boolean isUnitMode() { + return unitMode; + } + + + public void setUnitMode(boolean isUnitMode) { + this.unitMode = isUnitMode; + } + + + public long getAdjustThreadPoolNumsThreshold() { + return adjustThreadPoolNumsThreshold; + } + + + public void setAdjustThreadPoolNumsThreshold(long adjustThreadPoolNumsThreshold) { + this.adjustThreadPoolNumsThreshold = adjustThreadPoolNumsThreshold; + } + +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/MQConsumer.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/MQConsumer.java new file mode 100644 index 000000000..a74048f0a --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/MQConsumer.java @@ -0,0 +1,67 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.consumer; + +import java.util.Set; + +import com.alibaba.rocketmq.client.MQAdmin; +import com.alibaba.rocketmq.client.exception.MQBrokerException; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.common.message.MessageExt; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.remoting.exception.RemotingException; + + +/** + * Consumer接口 + * + * @author shijia.wxr + * @since 2013-7-24 + */ +public interface MQConsumer extends MQAdmin { + /** + * Consumer消费失败的消息可以选择重新发回到服务器端，并延时消费
+ * 会首先尝试将消息发回到消息之前存储的主机，此时只传送消息Offset，消息体不传送，不会占用网络带宽
+ * 如果发送失败，会自动重试发往其他主机，此时消息体也会传送
+ * 重传回去的消息只会被当前Consumer Group消费。 + * + * @param msg + * @param delayLevel + * @throws InterruptedException + * @throws MQBrokerException + * @throws RemotingException + * @throws MQClientException + */ + @Deprecated + public void sendMessageBack(final MessageExt msg, final int delayLevel) throws RemotingException, + MQBrokerException, InterruptedException, MQClientException; + + + public void sendMessageBack(final MessageExt msg, final int delayLevel, final String brokerName) + throws RemotingException, MQBrokerException, InterruptedException, MQClientException; + + + /** + * 根据topic获取对应的MessageQueue，是可被订阅的队列
+ * P.S 从Consumer Cache中拿数据，可以频繁调用。Cache中数据大约30秒更新一次 + * + * @param topic + * 消息Topic + * @return 返回队列集合 + * @throws MQClientException + */ + public Set fetchSubscribeMessageQueues(final String topic) throws MQClientException; +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/MQPullConsumer.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/MQPullConsumer.java new file mode 100644 index 000000000..ae8ffe097 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/MQPullConsumer.java @@ -0,0 +1,150 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.consumer; + +import java.util.Set; + +import com.alibaba.rocketmq.client.exception.MQBrokerException; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.remoting.exception.RemotingException; + + +/** + * 消费者，主动方式消费 + * + * @author shijia.wxr + * @since 2013-7-24 + */ +public interface MQPullConsumer extends MQConsumer { + /** + * 启动服务 + * + * @throws MQClientException + */ + public void start() throws MQClientException; + + + /** + * 关闭服务 + */ + public void shutdown(); + + + /** + * 注册监听队列变化的listener对象 + * + * @param topic + * @param listener + * 一旦发生变化，客户端会主动回调listener对象 + */ + public void registerMessageQueueListener(final String topic, final MessageQueueListener listener); + + + /** + * 指定队列，主动拉取消息，即使没有消息，也立刻返回 + * + * @param mq + * 指定具体要拉取的队列 + * @param subExpression + * 订阅过滤表达式字符串，broker依据此表达式进行过滤。目前只支持或运算
+ * eg: "tag1 || tag2 || tag3"
+ * 如果subExpression等于null或者*，则表示全部订阅 + * @param offset + * 从指定队列哪个位置开始拉取 + * @param maxNums + * 一次最多拉取条数 + * @return 参见PullResult + * @throws MQClientException + * @throws InterruptedException + * @throws MQBrokerException + * @throws RemotingException + */ + public PullResult pull(final MessageQueue mq, final String subExpression, final long offset, + final int maxNums) throws MQClientException, RemotingException, MQBrokerException, + InterruptedException; + + + public void pull(final MessageQueue mq, final String subExpression, final long offset, final int maxNums, + final PullCallback pullCallback) throws MQClientException, RemotingException, + InterruptedException; + + + /** + * 指定队列，主动拉取消息，如果没有消息，则broker阻塞一段时间再返回（时间可配置）
+ * broker阻塞期间，如果有消息，则立刻将消息返回 + * + * @param mq + * 指定具体要拉取的队列 + * @param subExpression + * 订阅过滤表达式字符串，broker依据此表达式进行过滤。目前只支持或运算
+ * eg: "tag1 || tag2 || tag3"
+ * 如果subExpression等于null或者*，则表示全部订阅 + * @param offset + * 从指定队列哪个位置开始拉取 + * @param maxNums + * 一次最多拉取条数 + * @return 参见PullResult + * @throws InterruptedException + * @throws MQBrokerException + * @throws RemotingException + * @throws MQClientException + */ + public PullResult pullBlockIfNotFound(final MessageQueue mq, final String subExpression, + final long offset, final int maxNums) throws MQClientException, RemotingException, + MQBrokerException, InterruptedException; + + + public void pullBlockIfNotFound(final MessageQueue mq, final String subExpression, final long offset, + final int maxNums, final PullCallback pullCallback) throws MQClientException, RemotingException, + InterruptedException; + + + /** + * 更新消费进度
+ * 只是更新Consumer缓存中的数据，如果是广播模式，则定时更新到本地存储
+ * 如果是集群模式，则定时更新到远端Broker
+ *

+ * P.S. 可频繁调用，无性能开销 + * + * @param mq + * @param offset + * @throws MQClientException + */ + public void updateConsumeOffset(final MessageQueue mq, final long offset) throws MQClientException; + + + /** + * 获取消费进度，返回-1表示出错 + * + * @param mq + * @param fromStore + * @return + * @throws MQClientException + */ + public long fetchConsumeOffset(final MessageQueue mq, final boolean fromStore) throws MQClientException; + + + /** + * 根据topic获取MessageQueue，以均衡方式在组内多个成员之间分配 + * + * @param topic + * 消息Topic + * @return 返回队列集合 + * @throws MQClientException + */ + public Set fetchMessageQueuesInBalance(final String topic) throws MQClientException; +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/MQPullConsumerScheduleService.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/MQPullConsumerScheduleService.java new file mode 100644 index 000000000..19125a42c --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/MQPullConsumerScheduleService.java @@ -0,0 +1,245 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.consumer; + +import java.util.Iterator; +import java.util.Map.Entry; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +import org.slf4j.Logger; + +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.client.log.ClientLogger; +import com.alibaba.rocketmq.common.ThreadFactoryImpl; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.common.protocol.heartbeat.MessageModel; + + +/** + * PullConsumer的调度服务，降低Pull方式的编程复杂度 + * + * @author shijia.wxr + * @since 2014-2-26 + */ +public class MQPullConsumerScheduleService { + private final Logger log = ClientLogger.getLog(); + private DefaultMQPullConsumer defaultMQPullConsumer; + private int pullThreadNums = 20; + private ConcurrentHashMap callbackTable = + new ConcurrentHashMap(); + + /** + * 具体实现，使用者不用关心 + */ + private ScheduledThreadPoolExecutor scheduledThreadPoolExecutor; + private final MessageQueueListener messageQueueListener = new MessageQueueListenerImpl(); + + // 正在拉取的任务 + private final ConcurrentHashMap taskTable = + new ConcurrentHashMap(); + + class MessageQueueListenerImpl implements MessageQueueListener { + @Override + public void messageQueueChanged(String topic, Set mqAll, Set mqDivided) { + MessageModel messageModel = + MQPullConsumerScheduleService.this.defaultMQPullConsumer.getMessageModel(); + switch (messageModel) { + case BROADCASTING: + MQPullConsumerScheduleService.this.putTask(topic, mqAll); + break; + case CLUSTERING: + MQPullConsumerScheduleService.this.putTask(topic, mqDivided); + break; + default: + break; + } + } + } + + class PullTaskImpl implements Runnable { + private final MessageQueue messageQueue; + private volatile boolean cancelled = false; + + + public PullTaskImpl(final MessageQueue messageQueue) { + this.messageQueue = messageQueue; + } + + + @Override + public void run() { + String topic = this.messageQueue.getTopic(); + if (!this.isCancelled()) { + PullTaskCallback pullTaskCallback = + MQPullConsumerScheduleService.this.callbackTable.get(topic); + if (pullTaskCallback != null) { + final PullTaskContext context = new PullTaskContext(); + context.setPullConsumer(MQPullConsumerScheduleService.this.defaultMQPullConsumer); + try { + pullTaskCallback.doPullTask(this.messageQueue, context); + } + catch (Throwable e) { + context.setPullNextDelayTimeMillis(1000); + log.error("doPullTask Exception", e); + } + + if (!this.isCancelled()) { + MQPullConsumerScheduleService.this.scheduledThreadPoolExecutor.schedule(this, + context.getPullNextDelayTimeMillis(), TimeUnit.MILLISECONDS); + } + else { + log.warn("The Pull Task is cancelled after doPullTask, {}", messageQueue); + } + } + else { + log.warn("Pull Task Callback not exist , {}", topic); + } + } + else { + log.warn("The Pull Task is cancelled, {}", messageQueue); + } + } + + + public boolean isCancelled() { + return cancelled; + } + + + public void setCancelled(boolean cancelled) { + this.cancelled = cancelled; + } + + + public MessageQueue getMessageQueue() { + return messageQueue; + } + } + + + public MQPullConsumerScheduleService(final String consumerGroup) { + this.defaultMQPullConsumer = new DefaultMQPullConsumer(consumerGroup); + this.defaultMQPullConsumer.setMessageModel(MessageModel.CLUSTERING); + } + + + public void putTask(String topic, Set mqNewSet) { + // 删除多余的队列 + Iterator> it = this.taskTable.entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + if (next.getKey().getTopic().equals(topic)) { + if (!mqNewSet.contains(next.getKey())) { + next.getValue().setCancelled(true); + it.remove(); + } + } + } + + // 增加新的队列 + for (MessageQueue mq : mqNewSet) { + if (!this.taskTable.containsKey(mq)) { + PullTaskImpl command = new PullTaskImpl(mq); + this.taskTable.put(mq, command); + this.scheduledThreadPoolExecutor.schedule(command, 0, TimeUnit.MILLISECONDS); + } + } + } + + + /** + * 启动服务 + * + * @throws MQClientException + */ + public void start() throws MQClientException { + final String group = this.defaultMQPullConsumer.getConsumerGroup(); + this.scheduledThreadPoolExecutor = new ScheduledThreadPoolExecutor(// + this.pullThreadNums,// + new ThreadFactoryImpl("PullMsgThread-" + group)// + ); + + this.defaultMQPullConsumer.setMessageQueueListener(this.messageQueueListener); + + this.defaultMQPullConsumer.start(); + + log.info("MQPullConsumerScheduleService start OK, {} {}", + this.defaultMQPullConsumer.getConsumerGroup(), this.callbackTable); + } + + + public void registerPullTaskCallback(final String topic, final PullTaskCallback callback) { + this.callbackTable.put(topic, callback); + this.defaultMQPullConsumer.registerMessageQueueListener(topic, null); + } + + + /** + * 关闭服务 + */ + public void shutdown() { + if (this.scheduledThreadPoolExecutor != null) { + this.scheduledThreadPoolExecutor.shutdown(); + } + + if (this.defaultMQPullConsumer != null) { + this.defaultMQPullConsumer.shutdown(); + } + } + + + public ConcurrentHashMap getCallbackTable() { + return callbackTable; + } + + + public void setCallbackTable(ConcurrentHashMap callbackTable) { + this.callbackTable = callbackTable; + } + + + public int getPullThreadNums() { + return pullThreadNums; + } + + + public void setPullThreadNums(int pullThreadNums) { + this.pullThreadNums = pullThreadNums; + } + + + public DefaultMQPullConsumer getDefaultMQPullConsumer() { + return defaultMQPullConsumer; + } + + + public void setDefaultMQPullConsumer(DefaultMQPullConsumer defaultMQPullConsumer) { + this.defaultMQPullConsumer = defaultMQPullConsumer; + } + + + public MessageModel getMessageModel() { + return this.defaultMQPullConsumer.getMessageModel(); + } + + + public void setMessageModel(MessageModel messageModel) { + this.defaultMQPullConsumer.setMessageModel(messageModel); + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/MQPushConsumer.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/MQPushConsumer.java new file mode 100644 index 000000000..b47d35249 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/MQPushConsumer.java @@ -0,0 +1,106 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.consumer; + +import com.alibaba.rocketmq.client.consumer.listener.MessageListener; +import com.alibaba.rocketmq.client.exception.MQClientException; + + +/** + * 消费者，被动方式消费 + * + * @author shijia.wxr + * @since 2013-7-24 + */ +public interface MQPushConsumer extends MQConsumer { + /** + * 启动服务，调用之前确保registerMessageListener与subscribe都已经调用
+ * 或者已经通过Spring注入了相关配置 + * + * @throws MQClientException + */ + public void start() throws MQClientException; + + + /** + * 关闭服务，一旦关闭，此对象将不可用 + */ + public void shutdown(); + + + /** + * 注册消息监听器，一个Consumer只能有一个监听器 + * + * @param messageListener + */ + public void registerMessageListener(final MessageListener messageListener); + + + /** + * 订阅消息，方法可以调用多次来订阅不同的Topic，也可覆盖之前Topic的订阅过滤表达式 + * + * @param topic + * 消息主题 + * @param subExpression + * 1、订阅过滤表达式字符串，broker依据此表达式进行过滤。目前只支持或运算
+ * 例如: "tag1 || tag2 || tag3"
+ * 如果subExpression等于null或者*，则表示全部订阅
+ * + * 2、高级过滤方式，传入一个Java程序，例如: + * "rocketmq.message.filter.cousumergroup.FilterClassName"
+ * "rocketmq.message.filter.cousumergroup.topic1.FilterClassName"
+ * 注意事项：
+ * a、Java程序必须继承于com.alibaba.rocketmq.common.filter.MessageFilter， + * 并实现相应的接口来过滤
+ * b、Java程序必须是UTF-8编码
+ * c、这个Java过滤程序只能依赖JDK里的类，非JDK的Java类一律不能依赖 + * d、过滤方法里不允许抛异常，只要抛异常，整个消费过程就停止 + * e、FilterClassName.java文件放置到CLASSPATH目录下，例如src/main/resources + * @param listener + * 消息回调监听器 + * @throws MQClientException + */ + public void subscribe(final String topic, final String subExpression) throws MQClientException; + + + /** + * 取消订阅，从当前订阅组内注销，消息会被订阅组内其他订阅者订阅 + * + * @param topic + * 消息主题 + */ + public void unsubscribe(final String topic); + + + /** + * 动态调整消费线程池线程数量 + * + * @param corePoolSize + */ + public void updateCorePoolSize(int corePoolSize); + + + /** + * 消费线程挂起，暂停消费 + */ + public void suspend(); + + + /** + * 消费线程恢复，继续消费 + */ + public void resume(); +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/MessageQueueListener.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/MessageQueueListener.java new file mode 100644 index 000000000..9d0c8db93 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/MessageQueueListener.java @@ -0,0 +1,32 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.consumer; + +import java.util.Set; + +import com.alibaba.rocketmq.common.message.MessageQueue; + + +/** + * 队列变化监听器 + * + * @author shijia.wxr + * @since 2013-7-24 + */ +public interface MessageQueueListener { + public void messageQueueChanged(final String topic, final Set mqAll, + final Set mqDivided); +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/PullCallback.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/PullCallback.java new file mode 100644 index 000000000..cde8811e4 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/PullCallback.java @@ -0,0 +1,29 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.consumer; + +/** + * 异步拉消息回调接口 + * + * @author shijia.wxr + * @since 2013-7-24 + */ +public interface PullCallback { + public void onSuccess(final PullResult pullResult); + + + public void onException(final Throwable e); +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/PullResult.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/PullResult.java new file mode 100644 index 000000000..fe04077d1 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/PullResult.java @@ -0,0 +1,84 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.consumer; + +import java.util.List; + +import com.alibaba.rocketmq.common.message.MessageExt; + + +/** + * 拉消息返回结果 + * + * @author shijia.wxr + * @since 2013-7-24 + */ +public class PullResult { + private final PullStatus pullStatus; + private final long nextBeginOffset; + private final long minOffset; + private final long maxOffset; + private List msgFoundList; + + + public PullResult(PullStatus pullStatus, long nextBeginOffset, long minOffset, long maxOffset, + List msgFoundList) { + super(); + this.pullStatus = pullStatus; + this.nextBeginOffset = nextBeginOffset; + this.minOffset = minOffset; + this.maxOffset = maxOffset; + this.msgFoundList = msgFoundList; + } + + + public PullStatus getPullStatus() { + return pullStatus; + } + + + public long getNextBeginOffset() { + return nextBeginOffset; + } + + + public long getMinOffset() { + return minOffset; + } + + + public long getMaxOffset() { + return maxOffset; + } + + + public List getMsgFoundList() { + return msgFoundList; + } + + + public void setMsgFoundList(List msgFoundList) { + this.msgFoundList = msgFoundList; + } + + + @Override + public String toString() { + return "PullResult [pullStatus=" + pullStatus + ", nextBeginOffset=" + nextBeginOffset + + ", minOffset=" + minOffset + ", maxOffset=" + maxOffset + ", msgFoundList=" + + (msgFoundList == null ? 0 : msgFoundList.size()) + "]"; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/PullStatus.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/PullStatus.java new file mode 100644 index 000000000..e4e9b3581 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/PullStatus.java @@ -0,0 +1,39 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.consumer; + +/** + * @author shijia.wxr + * @since 2013-7-24 + */ +public enum PullStatus { + /** + * 找到消息 + */ + FOUND, + /** + * 没有新的消息可以被拉取 + */ + NO_NEW_MSG, + /** + * 经过过滤后，没有匹配的消息 + */ + NO_MATCHED_MSG, + /** + * Offset不合法，可能过大或者过小 + */ + OFFSET_ILLEGAL +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/PullTaskCallback.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/PullTaskCallback.java new file mode 100644 index 000000000..0cdd569cc --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/PullTaskCallback.java @@ -0,0 +1,23 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.consumer; + +import com.alibaba.rocketmq.common.message.MessageQueue; + + +public interface PullTaskCallback { + public void doPullTask(final MessageQueue mq, final PullTaskContext context); +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/PullTaskContext.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/PullTaskContext.java new file mode 100644 index 000000000..179bd5846 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/PullTaskContext.java @@ -0,0 +1,45 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.consumer; + +public class PullTaskContext { + /** + * 距离下次拉取这个队列的间隔时间，单位毫秒 + */ + private int pullNextDelayTimeMillis = 200; + + private MQPullConsumer pullConsumer; + + + public int getPullNextDelayTimeMillis() { + return pullNextDelayTimeMillis; + } + + + public void setPullNextDelayTimeMillis(int pullNextDelayTimeMillis) { + this.pullNextDelayTimeMillis = pullNextDelayTimeMillis; + } + + + public MQPullConsumer getPullConsumer() { + return pullConsumer; + } + + + public void setPullConsumer(MQPullConsumer pullConsumer) { + this.pullConsumer = pullConsumer; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/listener/ConsumeConcurrentlyContext.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/listener/ConsumeConcurrentlyContext.java new file mode 100644 index 000000000..134a70223 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/listener/ConsumeConcurrentlyContext.java @@ -0,0 +1,73 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.consumer.listener; + +import com.alibaba.rocketmq.common.message.MessageQueue; + + +/** + * 消费消息上下文，同一队列的消息会并行消费，消息无顺序性 + * + * @author shijia.wxr + * @since 2013-7-24 + */ +public class ConsumeConcurrentlyContext { + /** + * 要消费的消息属于哪个队列 + */ + private final MessageQueue messageQueue; + /** + * 下次消息重试延时时间
+ * -1，表示不重试，直接进入死信队列
+ * 0，表示由服务器根据重试次数自动叠加
+ * >0，表示客户端强制指定延时Level + */ + private int delayLevelWhenNextConsume = 0; + /** + * 对于批量消费，ack至哪条消息，默认全部ack，至最后一条消息 + */ + private int ackIndex = Integer.MAX_VALUE; + + + public ConsumeConcurrentlyContext(MessageQueue messageQueue) { + this.messageQueue = messageQueue; + } + + + public int getDelayLevelWhenNextConsume() { + return delayLevelWhenNextConsume; + } + + + public void setDelayLevelWhenNextConsume(int delayLevelWhenNextConsume) { + this.delayLevelWhenNextConsume = delayLevelWhenNextConsume; + } + + + public MessageQueue getMessageQueue() { + return messageQueue; + } + + + public int getAckIndex() { + return ackIndex; + } + + + public void setAckIndex(int ackIndex) { + this.ackIndex = ackIndex; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/listener/ConsumeConcurrentlyStatus.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/listener/ConsumeConcurrentlyStatus.java new file mode 100644 index 000000000..eb73ac854 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/listener/ConsumeConcurrentlyStatus.java @@ -0,0 +1,29 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.consumer.listener; + +/** + * 并行消费，消费结果 + * + * @author shijia.wxr + * @since 2013-7-24 + */ +public enum ConsumeConcurrentlyStatus { + // 表示消费成功 + CONSUME_SUCCESS, + // 表示消费失败，但是稍后还会重新消费这批消息 + RECONSUME_LATER, +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/listener/ConsumeOrderlyContext.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/listener/ConsumeOrderlyContext.java new file mode 100644 index 000000000..e5de97229 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/listener/ConsumeOrderlyContext.java @@ -0,0 +1,70 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.consumer.listener; + +import com.alibaba.rocketmq.common.message.MessageQueue; + + +/** + * 消费消息上下文，同一队列的消息同一时刻只有一个线程消费，可保证同一队列消息顺序消费 + * + * @author shijia.wxr + * @since 2013-7-24 + */ +public class ConsumeOrderlyContext { + /** + * 要消费的消息属于哪个队列 + */ + private final MessageQueue messageQueue; + /** + * 消息Offset是否自动提交 + */ + private boolean autoCommit = true; + /** + * 将当前队列挂起时间，单位毫秒 + */ + private long suspendCurrentQueueTimeMillis = 1000; + + + public ConsumeOrderlyContext(MessageQueue messageQueue) { + this.messageQueue = messageQueue; + } + + + public boolean isAutoCommit() { + return autoCommit; + } + + + public void setAutoCommit(boolean autoCommit) { + this.autoCommit = autoCommit; + } + + + public MessageQueue getMessageQueue() { + return messageQueue; + } + + + public long getSuspendCurrentQueueTimeMillis() { + return suspendCurrentQueueTimeMillis; + } + + + public void setSuspendCurrentQueueTimeMillis(long suspendCurrentQueueTimeMillis) { + this.suspendCurrentQueueTimeMillis = suspendCurrentQueueTimeMillis; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/listener/ConsumeOrderlyStatus.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/listener/ConsumeOrderlyStatus.java new file mode 100644 index 000000000..8e3667ec3 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/listener/ConsumeOrderlyStatus.java @@ -0,0 +1,35 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.consumer.listener; + +/** + * 顺序消费，消费结果 + * + * @author shijia.wxr + * @since 2013-7-24 + */ +public enum ConsumeOrderlyStatus { + // 消息处理成功 + SUCCESS, + // 回滚消息（只供精卫binlog复制使用） + @Deprecated + ROLLBACK, + // 提交消息（只供精卫binlog复制使用） + @Deprecated + COMMIT, + // 将当前队列挂起一小会儿 + SUSPEND_CURRENT_QUEUE_A_MOMENT, +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/listener/MessageListener.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/listener/MessageListener.java new file mode 100644 index 000000000..00ca70d59 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/listener/MessageListener.java @@ -0,0 +1,26 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.consumer.listener; + +/** + * 消息监听器，被动方式订阅消息使用，需要用户实现
+ * 应用不可以直接继承此接口 + * + * @author shijia.wxr + * @since 2013-7-24 + */ +public interface MessageListener { +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/listener/MessageListenerConcurrently.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/listener/MessageListenerConcurrently.java new file mode 100644 index 000000000..59b8d9d4e --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/listener/MessageListenerConcurrently.java @@ -0,0 +1,42 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.consumer.listener; + +import java.util.List; + +import com.alibaba.rocketmq.common.message.MessageExt; + + +/** + * 同一队列的消息并行消费 + * + * @author shijia.wxr + * @since 2013-7-24 + */ +public interface MessageListenerConcurrently extends MessageListener { + /** + * 方法抛出异常等同于返回 ConsumeConcurrentlyStatus.RECONSUME_LATER
+ * P.S: 建议应用不要抛出异常 + * + * @param msgs + * msgs.size() >= 1
+ * DefaultMQPushConsumer.consumeMessageBatchMaxSize=1，默认消息数为1 + * @param context + * @return + */ + public ConsumeConcurrentlyStatus consumeMessage(final List msgs, + final ConsumeConcurrentlyContext context); +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/listener/MessageListenerOrderly.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/listener/MessageListenerOrderly.java new file mode 100644 index 000000000..0d9536082 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/listener/MessageListenerOrderly.java @@ -0,0 +1,42 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.consumer.listener; + +import java.util.List; + +import com.alibaba.rocketmq.common.message.MessageExt; + + +/** + * 同一队列的消息同一时刻只能一个线程消费，可保证消息在同一队列严格有序消费 + * + * @author shijia.wxr + * @since 2013-7-24 + */ +public interface MessageListenerOrderly extends MessageListener { + /** + * 方法抛出异常等同于返回 ConsumeOrderlyStatus.SUSPEND_CURRENT_QUEUE_A_MOMENT
+ * P.S: 建议应用不要抛出异常 + * + * @param msgs + * msgs.size() >= 1
+ * DefaultMQPushConsumer.consumeMessageBatchMaxSize=1，默认消息数为1 + * @param context + * @return + */ + public ConsumeOrderlyStatus consumeMessage(final List msgs, + final ConsumeOrderlyContext context); +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/rebalance/AllocateMessageQueueAveragely.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/rebalance/AllocateMessageQueueAveragely.java new file mode 100644 index 000000000..ce148c23b --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/rebalance/AllocateMessageQueueAveragely.java @@ -0,0 +1,79 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.consumer.rebalance; + +import java.util.ArrayList; +import java.util.List; + +import org.slf4j.Logger; + +import com.alibaba.rocketmq.client.consumer.AllocateMessageQueueStrategy; +import com.alibaba.rocketmq.client.log.ClientLogger; +import com.alibaba.rocketmq.common.message.MessageQueue; + + +/** + * 平均分配队列算法（块状） + * + * @author fuchong + * @author manhong.yqd + * @since 2013-7-24 + */ +public class AllocateMessageQueueAveragely implements AllocateMessageQueueStrategy { + private final Logger log = ClientLogger.getLog(); + + + @Override + public String getName() { + return "AVG"; + } + + + @Override + public List allocate(String consumerGroup, String currentCID, List mqAll, + List cidAll) { + if (currentCID == null || currentCID.length() < 1) { + throw new IllegalArgumentException("currentCID is empty"); + } + if (mqAll == null || mqAll.isEmpty()) { + throw new IllegalArgumentException("mqAll is null or mqAll empty"); + } + if (cidAll == null || cidAll.isEmpty()) { + throw new IllegalArgumentException("cidAll is null or cidAll empty"); + } + + List result = new ArrayList(); + if (!cidAll.contains(currentCID)) { // 不存在此ConsumerId ,直接返回 + log.info("[BUG] ConsumerGroup: {} The consumerId: {} not in cidAll: {}", // + consumerGroup, // + currentCID,// + cidAll); + return result; + } + + int index = cidAll.indexOf(currentCID); + int mod = mqAll.size() % cidAll.size(); + int averageSize = + mqAll.size() <= cidAll.size() ? 1 : (mod > 0 && index < mod ? mqAll.size() / cidAll.size() + + 1 : mqAll.size() / cidAll.size()); + int startIndex = (mod > 0 && index < mod) ? index * averageSize : index * averageSize + mod; + int range = Math.min(averageSize, mqAll.size() - startIndex); + for (int i = 0; i < range; i++) { + result.add(mqAll.get((startIndex + i) % mqAll.size())); + } + return result; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/rebalance/AllocateMessageQueueAveragelyByCircle.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/rebalance/AllocateMessageQueueAveragelyByCircle.java new file mode 100644 index 000000000..49aef6407 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/rebalance/AllocateMessageQueueAveragelyByCircle.java @@ -0,0 +1,74 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.consumer.rebalance; + +import java.util.ArrayList; +import java.util.List; + +import org.slf4j.Logger; + +import com.alibaba.rocketmq.client.consumer.AllocateMessageQueueStrategy; +import com.alibaba.rocketmq.client.log.ClientLogger; +import com.alibaba.rocketmq.common.message.MessageQueue; + + +/** + * 平均分配队列算法(环状） + * + * @author manhong.yqd + * @since 2014-09-10 + */ +public class AllocateMessageQueueAveragelyByCircle implements AllocateMessageQueueStrategy { + private final Logger log = ClientLogger.getLog(); + + + @Override + public String getName() { + return "AVG_BY_CIRCLE"; + } + + + @Override + public List allocate(String consumerGroup, String currentCID, List mqAll, + List cidAll) { + if (currentCID == null || currentCID.length() < 1) { + throw new IllegalArgumentException("currentCID is empty"); + } + if (mqAll == null || mqAll.isEmpty()) { + throw new IllegalArgumentException("mqAll is null or mqAll empty"); + } + if (cidAll == null || cidAll.isEmpty()) { + throw new IllegalArgumentException("cidAll is null or cidAll empty"); + } + + List result = new ArrayList(); + if (!cidAll.contains(currentCID)) { // 不存在此ConsumerId ,直接返回 + log.info("[BUG] ConsumerGroup: {} The consumerId: {} not in cidAll: {}", // + consumerGroup, // + currentCID,// + cidAll); + return result; + } + + int index = cidAll.indexOf(currentCID); + for (int i = index; i < mqAll.size(); i++) { + if (i % cidAll.size() == index) { + result.add(mqAll.get(i)); + } + } + return result; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/rebalance/AllocateMessageQueueByConfig.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/rebalance/AllocateMessageQueueByConfig.java new file mode 100644 index 000000000..a3380ed5d --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/rebalance/AllocateMessageQueueByConfig.java @@ -0,0 +1,55 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.consumer.rebalance; + +import java.util.List; + +import com.alibaba.rocketmq.client.consumer.AllocateMessageQueueStrategy; +import com.alibaba.rocketmq.common.message.MessageQueue; + + +/** + * 按照配置来分配队列，建议应用使用Spring来初始化 + * + * @author shijia.wxr + * @since 2013-7-24 + */ +public class AllocateMessageQueueByConfig implements AllocateMessageQueueStrategy { + private List messageQueueList; + + + @Override + public String getName() { + return "CONFIG"; + } + + + @Override + public List allocate(String consumerGroup, String currentCID, List mqAll, + List cidAll) { + return this.messageQueueList; + } + + + public List getMessageQueueList() { + return messageQueueList; + } + + + public void setMessageQueueList(List messageQueueList) { + this.messageQueueList = messageQueueList; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/rebalance/AllocateMessageQueueByMachineRoom.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/rebalance/AllocateMessageQueueByMachineRoom.java new file mode 100644 index 000000000..46c330938 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/rebalance/AllocateMessageQueueByMachineRoom.java @@ -0,0 +1,80 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.consumer.rebalance; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import com.alibaba.rocketmq.client.consumer.AllocateMessageQueueStrategy; +import com.alibaba.rocketmq.common.message.MessageQueue; + + +/** + * 按照机房来分配队列，例如支付宝逻辑机房 + * + * @author linye + * @since 2013-7-24 + */ +public class AllocateMessageQueueByMachineRoom implements AllocateMessageQueueStrategy { + private Set consumeridcs; + + + @Override + public String getName() { + return "MACHINE_ROOM"; + } + + + @Override + public List allocate(String consumerGroup, String currentCID, List mqAll, + List cidAll) { + List result = new ArrayList(); + int currentIndex = cidAll.indexOf(currentCID); + if (currentIndex < 0) { + return result; + } + List premqAll = new ArrayList(); + for (MessageQueue mq : mqAll) { + String[] temp = mq.getBrokerName().split("@"); + if (temp.length == 2 && consumeridcs.contains(temp[0])) { + premqAll.add(mq); + } + } + // Todo cid + int mod = premqAll.size() / cidAll.size(); + int rem = premqAll.size() % cidAll.size(); + int startindex = mod * currentIndex; + int endindex = startindex + mod; + for (int i = startindex; i < endindex; i++) { + result.add(mqAll.get(i)); + } + if (rem > currentIndex) { + result.add(premqAll.get(currentIndex + mod * cidAll.size())); + } + return result; + } + + + public Set getConsumeridcs() { + return consumeridcs; + } + + + public void setConsumeridcs(Set consumeridcs) { + this.consumeridcs = consumeridcs; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/store/LocalFileOffsetStore.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/store/LocalFileOffsetStore.java new file mode 100644 index 000000000..b531dba8d --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/store/LocalFileOffsetStore.java @@ -0,0 +1,235 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.consumer.store; + +import java.io.File; +import java.io.IOException; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; + +import org.slf4j.Logger; + +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.client.impl.factory.MQClientInstance; +import com.alibaba.rocketmq.client.log.ClientLogger; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.help.FAQUrl; +import com.alibaba.rocketmq.common.message.MessageQueue; + + +/** + * 消费进度存储到Consumer本地 + * + * @author shijia.wxr + * @since 2013-7-24 + */ +public class LocalFileOffsetStore implements OffsetStore { + public final static String LocalOffsetStoreDir = System.getProperty( + "rocketmq.client.localOffsetStoreDir", // + System.getProperty("user.home") + File.separator + ".rocketmq_offsets"); + private final static Logger log = ClientLogger.getLog(); + private final MQClientInstance mQClientFactory; + private final String groupName; + // 本地Offset存储路径 + private final String storePath; + private ConcurrentHashMap offsetTable = + new ConcurrentHashMap(); + + + public LocalFileOffsetStore(MQClientInstance mQClientFactory, String groupName) { + this.mQClientFactory = mQClientFactory; + this.groupName = groupName; + this.storePath = LocalOffsetStoreDir + File.separator + // + this.mQClientFactory.getClientId() + File.separator + // + this.groupName + File.separator + // + "offsets.json"; + } + + + @Override + public void load() throws MQClientException { + OffsetSerializeWrapper offsetSerializeWrapper = this.readLocalOffset(); + if (offsetSerializeWrapper != null && offsetSerializeWrapper.getOffsetTable() != null) { + offsetTable.putAll(offsetSerializeWrapper.getOffsetTable()); + + for (MessageQueue mq : offsetSerializeWrapper.getOffsetTable().keySet()) { + AtomicLong offset = offsetSerializeWrapper.getOffsetTable().get(mq); + log.info("load consumer's offset, {} {} {}",// + this.groupName,// + mq,// + offset.get()); + } + } + } + + + @Override + public void updateOffset(MessageQueue mq, long offset, boolean increaseOnly) { + if (mq != null) { + AtomicLong offsetOld = this.offsetTable.get(mq); + if (null == offsetOld) { + offsetOld = this.offsetTable.putIfAbsent(mq, new AtomicLong(offset)); + } + + if (null != offsetOld) { + if (increaseOnly) { + MixAll.compareAndIncreaseOnly(offsetOld, offset); + } + else { + offsetOld.set(offset); + } + } + } + } + + + @Override + public long readOffset(final MessageQueue mq, final ReadOffsetType type) { + if (mq != null) { + switch (type) { + case MEMORY_FIRST_THEN_STORE: + case READ_FROM_MEMORY: { + AtomicLong offset = this.offsetTable.get(mq); + if (offset != null) { + return offset.get(); + } + else if (ReadOffsetType.READ_FROM_MEMORY == type) { + return -1; + } + } + case READ_FROM_STORE: { + OffsetSerializeWrapper offsetSerializeWrapper; + try { + offsetSerializeWrapper = this.readLocalOffset(); + } + catch (MQClientException e) { + return -1; + } + if (offsetSerializeWrapper != null && offsetSerializeWrapper.getOffsetTable() != null) { + AtomicLong offset = offsetSerializeWrapper.getOffsetTable().get(mq); + if (offset != null) { + this.updateOffset(mq, offset.get(), false); + return offset.get(); + } + } + } + default: + break; + } + } + + return -1; + } + + + @Override + public void persistAll(Set mqs) { + if (null == mqs || mqs.isEmpty()) + return; + + OffsetSerializeWrapper offsetSerializeWrapper = new OffsetSerializeWrapper(); + for (MessageQueue mq : this.offsetTable.keySet()) { + if (mqs.contains(mq)) { + AtomicLong offset = this.offsetTable.get(mq); + offsetSerializeWrapper.getOffsetTable().put(mq, offset); + } + } + + String jsonString = offsetSerializeWrapper.toJson(true); + if (jsonString != null) { + try { + MixAll.string2File(jsonString, this.storePath); + } + catch (IOException e) { + log.error("persistAll consumer offset Exception, " + this.storePath, e); + } + } + } + + + @Override + public void persist(MessageQueue mq) { + } + + + private OffsetSerializeWrapper readLocalOffset() throws MQClientException { + String content = MixAll.file2String(this.storePath); + // 文件不存在，或者为空文件 + if (null == content || content.length() == 0) { + return this.readLocalOffsetBak(); + } + else { + OffsetSerializeWrapper offsetSerializeWrapper = null; + try { + offsetSerializeWrapper = + OffsetSerializeWrapper.fromJson(content, OffsetSerializeWrapper.class); + } + catch (Exception e) { + log.warn("readLocalOffset Exception, and try to correct", e); + return this.readLocalOffsetBak(); + } + + return offsetSerializeWrapper; + } + } + + + private OffsetSerializeWrapper readLocalOffsetBak() throws MQClientException { + String content = MixAll.file2String(this.storePath + ".bak"); + if (content != null && content.length() > 0) { + OffsetSerializeWrapper offsetSerializeWrapper = null; + try { + offsetSerializeWrapper = + OffsetSerializeWrapper.fromJson(content, OffsetSerializeWrapper.class); + } + catch (Exception e) { + log.warn("readLocalOffset Exception", e); + throw new MQClientException("readLocalOffset Exception, maybe fastjson version too low" // + + FAQUrl.suggestTodo(FAQUrl.LOAD_JSON_EXCEPTION), // + e); + } + return offsetSerializeWrapper; + } + + return null; + } + + + @Override + public void removeOffset(MessageQueue mq) { + // 消费进度存储到Consumer本地时暂不做 offset 清理 + } + + + @Override + public Map cloneOffsetTable(String topic) { + Map cloneOffsetTable = new HashMap(); + Iterator iterator = this.offsetTable.keySet().iterator(); + while (iterator.hasNext()) { + MessageQueue mq = iterator.next(); + if (!UtilAll.isBlank(topic) && !topic.equals(mq.getTopic())) { + continue; + } + cloneOffsetTable.put(mq, this.offsetTable.get(mq).get()); + } + return cloneOffsetTable; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/store/OffsetSerializeWrapper.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/store/OffsetSerializeWrapper.java new file mode 100644 index 000000000..7d1d95ee3 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/store/OffsetSerializeWrapper.java @@ -0,0 +1,44 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.consumer.store; + +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; + +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; + + +/** + * Offset持久化，json包装类 + * + * @author shijia.wxr + * @since 2013-7-25 + */ +public class OffsetSerializeWrapper extends RemotingSerializable { + private ConcurrentHashMap offsetTable = + new ConcurrentHashMap(); + + + public ConcurrentHashMap getOffsetTable() { + return offsetTable; + } + + + public void setOffsetTable(ConcurrentHashMap offsetTable) { + this.offsetTable = offsetTable; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/store/OffsetStore.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/store/OffsetStore.java new file mode 100644 index 000000000..58748b807 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/store/OffsetStore.java @@ -0,0 +1,71 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.consumer.store; + +import java.util.Map; +import java.util.Set; + +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.common.message.MessageQueue; + + +/** + * Consumer Offset存储接口 + * + * @author shijia.wxr + * @since 2013-7-25 + */ +public interface OffsetStore { + /** + * 加载Offset + * + * @throws MQClientException + */ + public void load() throws MQClientException; + + + /** + * 更新消费进度，存储到内存 + */ + public void updateOffset(final MessageQueue mq, final long offset, final boolean increaseOnly); + + + /** + * 从本地缓存读取消费进度 + */ + public long readOffset(final MessageQueue mq, final ReadOffsetType type); + + + /** + * 持久化全部消费进度，可能持久化本地或者远端Broker + */ + public void persistAll(final Set mqs); + + + public void persist(final MessageQueue mq); + + + /** + * 删除不必要的MessageQueue offset + */ + public void removeOffset(MessageQueue mq); + + + /** + * 如果 topic 为空，则不对 topic 进行过滤，全部拷贝。 + */ + public Map cloneOffsetTable(String topic); +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/store/ReadOffsetType.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/store/ReadOffsetType.java new file mode 100644 index 000000000..9a485df32 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/store/ReadOffsetType.java @@ -0,0 +1,25 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.consumer.store; + +public enum ReadOffsetType { + // 只从Memory读取 + READ_FROM_MEMORY, + // 只从存储层读取（本地或者远端） + READ_FROM_STORE, + // 先从内存读，内存不存在再从存储层读 + MEMORY_FIRST_THEN_STORE, +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/store/RemoteBrokerOffsetStore.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/store/RemoteBrokerOffsetStore.java new file mode 100644 index 000000000..1b1a0d2f2 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/consumer/store/RemoteBrokerOffsetStore.java @@ -0,0 +1,262 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.consumer.store; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; + +import org.slf4j.Logger; + +import com.alibaba.rocketmq.client.exception.MQBrokerException; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.client.impl.FindBrokerResult; +import com.alibaba.rocketmq.client.impl.factory.MQClientInstance; +import com.alibaba.rocketmq.client.log.ClientLogger; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.common.protocol.header.QueryConsumerOffsetRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.UpdateConsumerOffsetRequestHeader; +import com.alibaba.rocketmq.remoting.exception.RemotingException; + + +/** + * 消费进度存储到远端Broker，比较可靠 + * + * @author shijia.wxr + * @since 2013-7-24 + */ +public class RemoteBrokerOffsetStore implements OffsetStore { + private final static Logger log = ClientLogger.getLog(); + private final MQClientInstance mQClientFactory; + private final String groupName; + private final AtomicLong storeTimesTotal = new AtomicLong(0); + private ConcurrentHashMap offsetTable = + new ConcurrentHashMap(); + + + public RemoteBrokerOffsetStore(MQClientInstance mQClientFactory, String groupName) { + this.mQClientFactory = mQClientFactory; + this.groupName = groupName; + } + + + @Override + public void load() { + } + + + @Override + public void updateOffset(MessageQueue mq, long offset, boolean increaseOnly) { + if (mq != null) { + AtomicLong offsetOld = this.offsetTable.get(mq); + if (null == offsetOld) { + offsetOld = this.offsetTable.putIfAbsent(mq, new AtomicLong(offset)); + } + + if (null != offsetOld) { + if (increaseOnly) { + MixAll.compareAndIncreaseOnly(offsetOld, offset); + } + else { + offsetOld.set(offset); + } + } + } + } + + + @Override + public long readOffset(final MessageQueue mq, final ReadOffsetType type) { + if (mq != null) { + switch (type) { + case MEMORY_FIRST_THEN_STORE: + case READ_FROM_MEMORY: { + AtomicLong offset = this.offsetTable.get(mq); + if (offset != null) { + return offset.get(); + } + else if (ReadOffsetType.READ_FROM_MEMORY == type) { + return -1; + } + } + case READ_FROM_STORE: { + try { + long brokerOffset = this.fetchConsumeOffsetFromBroker(mq); + AtomicLong offset = new AtomicLong(brokerOffset); + this.updateOffset(mq, offset.get(), false); + return brokerOffset; + } + // 当前订阅组在服务器没有对应的Offset + catch (MQBrokerException e) { + return -1; + } + // 其他通信错误 + catch (Exception e) { + log.warn("fetchConsumeOffsetFromBroker exception, " + mq, e); + return -2; + } + } + default: + break; + } + } + + return -1; + } + + + @Override + public void persistAll(Set mqs) { + if (null == mqs || mqs.isEmpty()) + return; + + final HashSet unusedMQ = new HashSet(); + long times = this.storeTimesTotal.getAndIncrement(); + + if (mqs != null && !mqs.isEmpty()) { + for (MessageQueue mq : this.offsetTable.keySet()) { + AtomicLong offset = this.offsetTable.get(mq); + if (offset != null) { + if (mqs.contains(mq)) { + try { + this.updateConsumeOffsetToBroker(mq, offset.get()); + // 每隔1分钟打印一次消费进度 + if ((times % 12) == 0) { + log.info("Group: {} ClientId: {} updateConsumeOffsetToBroker {} {}", // + this.groupName,// + this.mQClientFactory.getClientId(),// + mq, // + offset.get()); + } + } + catch (Exception e) { + log.error("updateConsumeOffsetToBroker exception, " + mq.toString(), e); + } + } + // 本地多余的队列，需要删除掉 + else { + unusedMQ.add(mq); + } + } + } + } + + if (!unusedMQ.isEmpty()) { + for (MessageQueue mq : unusedMQ) { + this.offsetTable.remove(mq); + log.info("remove unused mq, {}, {}", mq, this.groupName); + } + } + } + + + @Override + public void persist(MessageQueue mq) { + AtomicLong offset = this.offsetTable.get(mq); + if (offset != null) { + try { + this.updateConsumeOffsetToBroker(mq, offset.get()); + log.debug("updateConsumeOffsetToBroker {} {}", mq, offset.get()); + } + catch (Exception e) { + log.error("updateConsumeOffsetToBroker exception, " + mq.toString(), e); + } + } + } + + + /** + * 更新Consumer Offset，在Master断网期间，可能会更新到Slave，这里需要优化，或者在Slave端优化， TODO + */ + private void updateConsumeOffsetToBroker(MessageQueue mq, long offset) throws RemotingException, + MQBrokerException, InterruptedException, MQClientException { + FindBrokerResult findBrokerResult = this.mQClientFactory.findBrokerAddressInAdmin(mq.getBrokerName()); + if (null == findBrokerResult) { + // TODO 此处可能对Name Server压力过大，需要调优 + this.mQClientFactory.updateTopicRouteInfoFromNameServer(mq.getTopic()); + findBrokerResult = this.mQClientFactory.findBrokerAddressInAdmin(mq.getBrokerName()); + } + + if (findBrokerResult != null) { + UpdateConsumerOffsetRequestHeader requestHeader = new UpdateConsumerOffsetRequestHeader(); + requestHeader.setTopic(mq.getTopic()); + requestHeader.setConsumerGroup(this.groupName); + requestHeader.setQueueId(mq.getQueueId()); + requestHeader.setCommitOffset(offset); + + // 使用oneway形式，原因是服务器在删除文件时，这个调用可能会超时 + this.mQClientFactory.getMQClientAPIImpl().updateConsumerOffsetOneway( + findBrokerResult.getBrokerAddr(), requestHeader, 1000 * 5); + } + else { + throw new MQClientException("The broker[" + mq.getBrokerName() + "] not exist", null); + } + } + + + private long fetchConsumeOffsetFromBroker(MessageQueue mq) throws RemotingException, MQBrokerException, + InterruptedException, MQClientException { + FindBrokerResult findBrokerResult = this.mQClientFactory.findBrokerAddressInAdmin(mq.getBrokerName()); + if (null == findBrokerResult) { + // TODO 此处可能对Name Server压力过大，需要调优 + this.mQClientFactory.updateTopicRouteInfoFromNameServer(mq.getTopic()); + findBrokerResult = this.mQClientFactory.findBrokerAddressInAdmin(mq.getBrokerName()); + } + + if (findBrokerResult != null) { + QueryConsumerOffsetRequestHeader requestHeader = new QueryConsumerOffsetRequestHeader(); + requestHeader.setTopic(mq.getTopic()); + requestHeader.setConsumerGroup(this.groupName); + requestHeader.setQueueId(mq.getQueueId()); + + return this.mQClientFactory.getMQClientAPIImpl().queryConsumerOffset( + findBrokerResult.getBrokerAddr(), requestHeader, 1000 * 5); + } + else { + throw new MQClientException("The broker[" + mq.getBrokerName() + "] not exist", null); + } + } + + + public void removeOffset(MessageQueue mq) { + if (mq != null) { + this.offsetTable.remove(mq); + log.info("remove unnecessary messageQueue offset. mq={}, offsetTableSize={}", mq, + offsetTable.size()); + } + } + + + @Override + public Map cloneOffsetTable(String topic) { + Map cloneOffsetTable = new HashMap(); + Iterator iterator = this.offsetTable.keySet().iterator(); + while (iterator.hasNext()) { + MessageQueue mq = iterator.next(); + if (!UtilAll.isBlank(topic) && !topic.equals(mq.getTopic())) { + continue; + } + cloneOffsetTable.put(mq, this.offsetTable.get(mq).get()); + } + return cloneOffsetTable; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/exception/MQBrokerException.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/exception/MQBrokerException.java new file mode 100644 index 000000000..171c983c7 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/exception/MQBrokerException.java @@ -0,0 +1,50 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.exception; + +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.help.FAQUrl; + + +/** + * Broker异常 + * + * @author shijia.wxr + * @since 2013-7-24 + */ +public class MQBrokerException extends Exception { + private static final long serialVersionUID = 5975020272601250368L; + private final int responseCode; + private final String errorMessage; + + + public MQBrokerException(int responseCode, String errorMessage) { + super(FAQUrl.attachDefaultURL("CODE: " + UtilAll.responseCode2String(responseCode) + " DESC: " + + errorMessage)); + this.responseCode = responseCode; + this.errorMessage = errorMessage; + } + + + public int getResponseCode() { + return responseCode; + } + + + public String getErrorMessage() { + return errorMessage; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/exception/MQClientException.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/exception/MQClientException.java new file mode 100644 index 000000000..20259f64c --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/exception/MQClientException.java @@ -0,0 +1,56 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.exception; + +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.help.FAQUrl; + + +/** + * MQ异常类 + * + * @author shijia.wxr + */ +public class MQClientException extends Exception { + private static final long serialVersionUID = -5758410930844185841L; + private final int responseCode; + private final String errorMessage; + + + public MQClientException(String errorMessage, Throwable cause) { + super(FAQUrl.attachDefaultURL(errorMessage), cause); + this.responseCode = -1; + this.errorMessage = errorMessage; + } + + + public MQClientException(int responseCode, String errorMessage) { + super(FAQUrl.attachDefaultURL("CODE: " + UtilAll.responseCode2String(responseCode) + " DESC: " + + errorMessage)); + this.responseCode = responseCode; + this.errorMessage = errorMessage; + } + + + public int getResponseCode() { + return responseCode; + } + + + public String getErrorMessage() { + return errorMessage; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/hook/CheckForbiddenContext.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/hook/CheckForbiddenContext.java new file mode 100644 index 000000000..c07df8bbd --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/hook/CheckForbiddenContext.java @@ -0,0 +1,150 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.hook; + +import com.alibaba.rocketmq.client.impl.CommunicationMode; +import com.alibaba.rocketmq.client.producer.SendResult; +import com.alibaba.rocketmq.common.message.Message; +import com.alibaba.rocketmq.common.message.MessageQueue; + + +/** + * 权限控制时用 + * + * @author manhong.yqd + * @since 2014-3-19 + */ +public class CheckForbiddenContext { + private String nameSrvAddr; + private String group; + private Message message; + private MessageQueue mq; + private String brokerAddr; + private CommunicationMode communicationMode; + private SendResult sendResult; + private Exception exception; + private Object arg; + private boolean unitMode = false; + + + public String getGroup() { + return group; + } + + + public void setGroup(String group) { + this.group = group; + } + + + public Message getMessage() { + return message; + } + + + public void setMessage(Message message) { + this.message = message; + } + + + public MessageQueue getMq() { + return mq; + } + + + public void setMq(MessageQueue mq) { + this.mq = mq; + } + + + public String getBrokerAddr() { + return brokerAddr; + } + + + public void setBrokerAddr(String brokerAddr) { + this.brokerAddr = brokerAddr; + } + + + public CommunicationMode getCommunicationMode() { + return communicationMode; + } + + + public void setCommunicationMode(CommunicationMode communicationMode) { + this.communicationMode = communicationMode; + } + + + public SendResult getSendResult() { + return sendResult; + } + + + public void setSendResult(SendResult sendResult) { + this.sendResult = sendResult; + } + + + public Exception getException() { + return exception; + } + + + public void setException(Exception exception) { + this.exception = exception; + } + + + public Object getArg() { + return arg; + } + + + public void setArg(Object arg) { + this.arg = arg; + } + + + public boolean isUnitMode() { + return unitMode; + } + + + public void setUnitMode(boolean isUnitMode) { + this.unitMode = isUnitMode; + } + + + public String getNameSrvAddr() { + return nameSrvAddr; + } + + + public void setNameSrvAddr(String nameSrvAddr) { + this.nameSrvAddr = nameSrvAddr; + } + + + @Override + public String toString() { + return "SendMessageContext [nameSrvAddr=" + nameSrvAddr + ", group=" + group + ", message=" + message + + ", mq=" + mq + ", brokerAddr=" + brokerAddr + ", communicationMode=" + communicationMode + + ", sendResult=" + sendResult + ", exception=" + exception + ", unitMode=" + unitMode + + ", arg=" + arg + "]"; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/hook/CheckForbiddenHook.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/hook/CheckForbiddenHook.java new file mode 100644 index 000000000..94bfd67db --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/hook/CheckForbiddenHook.java @@ -0,0 +1,17 @@ +package com.alibaba.rocketmq.client.hook; + +import com.alibaba.rocketmq.client.exception.MQClientException; + + +/** + * 读写权限控制 Hook + * + * @author: manhong.yqd + * @since: 14-4-9 + */ +public interface CheckForbiddenHook { + public String hookName(); + + + public void checkForbidden(final CheckForbiddenContext context) throws MQClientException; +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/hook/ConsumeMessageContext.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/hook/ConsumeMessageContext.java new file mode 100644 index 000000000..2c288bd01 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/hook/ConsumeMessageContext.java @@ -0,0 +1,103 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.hook; + +import java.util.List; +import java.util.Map; + +import com.alibaba.rocketmq.common.message.MessageExt; +import com.alibaba.rocketmq.common.message.MessageQueue; + + +public class ConsumeMessageContext { + private String consumerGroup; + private List msgList; + private MessageQueue mq; + private boolean success; + private String status; + private Object mqTraceContext; + private Map props; + + + public String getConsumerGroup() { + return consumerGroup; + } + + + public void setConsumerGroup(String consumerGroup) { + this.consumerGroup = consumerGroup; + } + + + public List getMsgList() { + return msgList; + } + + + public void setMsgList(List msgList) { + this.msgList = msgList; + } + + + public MessageQueue getMq() { + return mq; + } + + + public void setMq(MessageQueue mq) { + this.mq = mq; + } + + + public boolean isSuccess() { + return success; + } + + + public void setSuccess(boolean success) { + this.success = success; + } + + + public Object getMqTraceContext() { + return mqTraceContext; + } + + + public void setMqTraceContext(Object mqTraceContext) { + this.mqTraceContext = mqTraceContext; + } + + + public Map getProps() { + return props; + } + + + public void setProps(Map props) { + this.props = props; + } + + + public String getStatus() { + return status; + } + + + public void setStatus(String status) { + this.status = status; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/hook/ConsumeMessageHook.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/hook/ConsumeMessageHook.java new file mode 100644 index 000000000..2fc1993f5 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/hook/ConsumeMessageHook.java @@ -0,0 +1,26 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.hook; + +public interface ConsumeMessageHook { + public String hookName(); + + + public void consumeMessageBefore(final ConsumeMessageContext context); + + + public void consumeMessageAfter(final ConsumeMessageContext context); +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/hook/FilterMessageContext.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/hook/FilterMessageContext.java new file mode 100644 index 000000000..c2a552528 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/hook/FilterMessageContext.java @@ -0,0 +1,93 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.hook; + +import java.util.List; + +import com.alibaba.rocketmq.common.message.MessageExt; +import com.alibaba.rocketmq.common.message.MessageQueue; + + +/** + * 消息过滤用 + * + * @author manhong.yqd + * @since 2014-3-19 + */ +public class FilterMessageContext { + private String consumerGroup; + private List msgList; + private MessageQueue mq; + private Object arg; + private boolean unitMode; + + + public String getConsumerGroup() { + return consumerGroup; + } + + + public void setConsumerGroup(String consumerGroup) { + this.consumerGroup = consumerGroup; + } + + + public List getMsgList() { + return msgList; + } + + + public void setMsgList(List msgList) { + this.msgList = msgList; + } + + + public MessageQueue getMq() { + return mq; + } + + + public void setMq(MessageQueue mq) { + this.mq = mq; + } + + + public Object getArg() { + return arg; + } + + + public void setArg(Object arg) { + this.arg = arg; + } + + + public boolean isUnitMode() { + return unitMode; + } + + + public void setUnitMode(boolean isUnitMode) { + this.unitMode = isUnitMode; + } + + + @Override + public String toString() { + return "ConsumeMessageContext [consumerGroup=" + consumerGroup + ", msgList=" + msgList + ", mq=" + + mq + ", arg=" + arg + "]"; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/hook/FilterMessageHook.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/hook/FilterMessageHook.java new file mode 100644 index 000000000..ccda1920b --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/hook/FilterMessageHook.java @@ -0,0 +1,29 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.hook; + +/** + * 消息过滤 Hook + * + * @author manhong.yqd + * @since 2014-3-19 + */ +public interface FilterMessageHook { + public String hookName(); + + + public void filterMessage(final FilterMessageContext context); +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/hook/SendMessageContext.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/hook/SendMessageContext.java new file mode 100644 index 000000000..b69cb3285 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/hook/SendMessageContext.java @@ -0,0 +1,137 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.hook; + +import java.util.Map; + +import com.alibaba.rocketmq.client.impl.CommunicationMode; +import com.alibaba.rocketmq.client.producer.SendResult; +import com.alibaba.rocketmq.common.message.Message; +import com.alibaba.rocketmq.common.message.MessageQueue; + + +public class SendMessageContext { + private String producerGroup; + private Message message; + private MessageQueue mq; + private String brokerAddr; + private String bornHost; + private CommunicationMode communicationMode; + private SendResult sendResult; + private Exception exception; + private Object mqTraceContext; + private Map props; + + + public String getProducerGroup() { + return producerGroup; + } + + + public void setProducerGroup(String producerGroup) { + this.producerGroup = producerGroup; + } + + + public Message getMessage() { + return message; + } + + + public void setMessage(Message message) { + this.message = message; + } + + + public MessageQueue getMq() { + return mq; + } + + + public void setMq(MessageQueue mq) { + this.mq = mq; + } + + + public String getBrokerAddr() { + return brokerAddr; + } + + + public void setBrokerAddr(String brokerAddr) { + this.brokerAddr = brokerAddr; + } + + + public CommunicationMode getCommunicationMode() { + return communicationMode; + } + + + public void setCommunicationMode(CommunicationMode communicationMode) { + this.communicationMode = communicationMode; + } + + + public SendResult getSendResult() { + return sendResult; + } + + + public void setSendResult(SendResult sendResult) { + this.sendResult = sendResult; + } + + + public Exception getException() { + return exception; + } + + + public void setException(Exception exception) { + this.exception = exception; + } + + + public Object getMqTraceContext() { + return mqTraceContext; + } + + + public void setMqTraceContext(Object mqTraceContext) { + this.mqTraceContext = mqTraceContext; + } + + + public Map getProps() { + return props; + } + + + public void setProps(Map props) { + this.props = props; + } + + + public String getBornHost() { + return bornHost; + } + + + public void setBornHost(String bornHost) { + this.bornHost = bornHost; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/hook/SendMessageHook.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/hook/SendMessageHook.java new file mode 100644 index 000000000..ee89baf35 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/hook/SendMessageHook.java @@ -0,0 +1,26 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.hook; + +public interface SendMessageHook { + public String hookName(); + + + public void sendMessageBefore(final SendMessageContext context); + + + public void sendMessageAfter(final SendMessageContext context); +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/ClientRemotingProcessor.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/ClientRemotingProcessor.java new file mode 100644 index 000000000..1522afef3 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/ClientRemotingProcessor.java @@ -0,0 +1,244 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.impl; + +import io.netty.channel.ChannelHandlerContext; + +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.Map; + +import org.slf4j.Logger; + +import com.alibaba.rocketmq.client.impl.factory.MQClientInstance; +import com.alibaba.rocketmq.client.impl.producer.MQProducerInner; +import com.alibaba.rocketmq.client.log.ClientLogger; +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.message.MessageConst; +import com.alibaba.rocketmq.common.message.MessageDecoder; +import com.alibaba.rocketmq.common.message.MessageExt; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.common.protocol.RequestCode; +import com.alibaba.rocketmq.common.protocol.ResponseCode; +import com.alibaba.rocketmq.common.protocol.body.ConsumeMessageDirectlyResult; +import com.alibaba.rocketmq.common.protocol.body.ConsumerRunningInfo; +import com.alibaba.rocketmq.common.protocol.body.GetConsumerStatusBody; +import com.alibaba.rocketmq.common.protocol.body.ResetOffsetBody; +import com.alibaba.rocketmq.common.protocol.header.CheckTransactionStateRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.ConsumeMessageDirectlyResultRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.GetConsumerRunningInfoRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.GetConsumerStatusRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.NotifyConsumerIdsChangedRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.ResetOffsetRequestHeader; +import com.alibaba.rocketmq.remoting.common.RemotingHelper; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; +import com.alibaba.rocketmq.remoting.netty.NettyRequestProcessor; +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; + + +/** + * Client接收Broker的回调操作，例如事务回调，或者其他管理类命令回调 + * + * @author shijia.wxr + * @since 2013-7-24 + */ +public class ClientRemotingProcessor implements NettyRequestProcessor { + private final Logger log = ClientLogger.getLog(); + private final MQClientInstance mqClientFactory; + + + public ClientRemotingProcessor(final MQClientInstance mqClientFactory) { + this.mqClientFactory = mqClientFactory; + } + + + @Override + public RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + switch (request.getCode()) { + case RequestCode.CHECK_TRANSACTION_STATE: + return this.checkTransactionState(ctx, request); + case RequestCode.NOTIFY_CONSUMER_IDS_CHANGED: + return this.notifyConsumerIdsChanged(ctx, request); + case RequestCode.RESET_CONSUMER_CLIENT_OFFSET: + return this.resetOffset(ctx, request); + case RequestCode.GET_CONSUMER_STATUS_FROM_CLIENT: + return this.getConsumeStatus(ctx, request); + + case RequestCode.GET_CONSUMER_RUNNING_INFO: + return this.getConsumerRunningInfo(ctx, request); + + case RequestCode.CONSUME_MESSAGE_DIRECTLY: + return this.consumeMessageDirectly(ctx, request); + default: + break; + } + return null; + } + + + private RemotingCommand consumeMessageDirectly(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + final ConsumeMessageDirectlyResultRequestHeader requestHeader = + (ConsumeMessageDirectlyResultRequestHeader) request + .decodeCommandCustomHeader(ConsumeMessageDirectlyResultRequestHeader.class); + + final MessageExt msg = MessageDecoder.decode(ByteBuffer.wrap(request.getBody())); + + ConsumeMessageDirectlyResult result = + this.mqClientFactory.consumeMessageDirectly(msg, requestHeader.getConsumerGroup(), + requestHeader.getBrokerName()); + + if (null != result) { + response.setCode(ResponseCode.SUCCESS); + response.setBody(result.encode()); + } + else { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark(String.format("The Consumer Group <%s> not exist in this consumer", + requestHeader.getConsumerGroup())); + } + + return response; + } + + + private RemotingCommand getConsumerRunningInfo(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + final GetConsumerRunningInfoRequestHeader requestHeader = + (GetConsumerRunningInfoRequestHeader) request + .decodeCommandCustomHeader(GetConsumerRunningInfoRequestHeader.class); + + ConsumerRunningInfo consumerRunningInfo = + this.mqClientFactory.consumerRunningInfo(requestHeader.getConsumerGroup()); + if (null != consumerRunningInfo) { + if (requestHeader.isJstackEnable()) { + String jstack = UtilAll.jstack(); + consumerRunningInfo.setJstack(jstack); + } + + response.setCode(ResponseCode.SUCCESS); + response.setBody(consumerRunningInfo.encode()); + } + else { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark(String.format("The Consumer Group <%s> not exist in this consumer", + requestHeader.getConsumerGroup())); + } + + return response; + } + + + /** + * Oneway调用，无返回值 + */ + public RemotingCommand checkTransactionState(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + final CheckTransactionStateRequestHeader requestHeader = + (CheckTransactionStateRequestHeader) request + .decodeCommandCustomHeader(CheckTransactionStateRequestHeader.class); + final ByteBuffer byteBuffer = ByteBuffer.wrap(request.getBody()); + final MessageExt messageExt = MessageDecoder.decode(byteBuffer); + if (messageExt != null) { + final String group = messageExt.getProperty(MessageConst.PROPERTY_PRODUCER_GROUP); + if (group != null) { + MQProducerInner producer = this.mqClientFactory.selectProducer(group); + if (producer != null) { + final String addr = RemotingHelper.parseChannelRemoteAddr(ctx.channel()); + producer.checkTransactionState(addr, messageExt, requestHeader); + } + else { + log.debug("checkTransactionState, pick producer by group[{}] failed", group); + } + } + else { + log.warn("checkTransactionState, pick producer group failed"); + } + } + else { + log.warn("checkTransactionState, decode message failed"); + } + + return null; + } + + + /** + * Oneway调用，无返回值 + */ + public RemotingCommand notifyConsumerIdsChanged(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + try { + final NotifyConsumerIdsChangedRequestHeader requestHeader = + (NotifyConsumerIdsChangedRequestHeader) request + .decodeCommandCustomHeader(NotifyConsumerIdsChangedRequestHeader.class); + log.info( + "receive broker's notification[{}], the consumer group: {} changed, rebalance immediately",// + RemotingHelper.parseChannelRemoteAddr(ctx.channel()),// + requestHeader.getConsumerGroup()); + this.mqClientFactory.rebalanceImmediately(); + } + catch (Exception e) { + log.error("notifyConsumerIdsChanged exception", RemotingHelper.exceptionSimpleDesc(e)); + } + return null; + } + + + /** + * 重置 offset， oneWay调用，无返回值。 + */ + public RemotingCommand resetOffset(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + final ResetOffsetRequestHeader requestHeader = + (ResetOffsetRequestHeader) request.decodeCommandCustomHeader(ResetOffsetRequestHeader.class); + log.info( + "invoke reset offset operation from broker. brokerAddr={}, topic={}, group={}, timestamp={}", + new Object[] { RemotingHelper.parseChannelRemoteAddr(ctx.channel()), requestHeader.getTopic(), + requestHeader.getGroup(), requestHeader.getTimestamp() }); + Map offsetTable = new HashMap(); + if (request.getBody() != null) { + ResetOffsetBody body = ResetOffsetBody.decode(request.getBody(), ResetOffsetBody.class); + offsetTable = body.getOffsetTable(); + } + this.mqClientFactory.resetOffset(requestHeader.getTopic(), requestHeader.getGroup(), offsetTable); + return null; + } + + + /** + * 获取 consumer 消息消费状态。 + */ + @Deprecated + public RemotingCommand getConsumeStatus(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + final GetConsumerStatusRequestHeader requestHeader = + (GetConsumerStatusRequestHeader) request + .decodeCommandCustomHeader(GetConsumerStatusRequestHeader.class); + + Map offsetTable = + this.mqClientFactory.getConsumerStatus(requestHeader.getTopic(), requestHeader.getGroup()); + GetConsumerStatusBody body = new GetConsumerStatusBody(); + body.setMessageQueueTable(offsetTable); + response.setBody(body.encode()); + response.setCode(ResponseCode.SUCCESS); + return response; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/CommunicationMode.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/CommunicationMode.java new file mode 100644 index 000000000..a06350411 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/CommunicationMode.java @@ -0,0 +1,28 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.impl; + +/** + * 通信方式 + * + * @author shijia.wxr + * @since 2013-7-24 + */ +public enum CommunicationMode { + SYNC, + ASYNC, + ONEWAY +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/FindBrokerResult.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/FindBrokerResult.java new file mode 100644 index 000000000..89ecee4eb --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/FindBrokerResult.java @@ -0,0 +1,41 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.impl; + +/** + * @author shijia.wxr + * @since 2013-7-24 + */ +public class FindBrokerResult { + private final String brokerAddr; + private final boolean slave; + + + public FindBrokerResult(String brokerAddr, boolean slave) { + this.brokerAddr = brokerAddr; + this.slave = slave; + } + + + public String getBrokerAddr() { + return brokerAddr; + } + + + public boolean isSlave() { + return slave; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/MQAdminImpl.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/MQAdminImpl.java new file mode 100644 index 000000000..139050af7 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/MQAdminImpl.java @@ -0,0 +1,404 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.impl; + +import java.net.UnknownHostException; +import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import org.slf4j.Logger; + +import com.alibaba.rocketmq.client.QueryResult; +import com.alibaba.rocketmq.client.exception.MQBrokerException; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.client.impl.factory.MQClientInstance; +import com.alibaba.rocketmq.client.impl.producer.TopicPublishInfo; +import com.alibaba.rocketmq.client.log.ClientLogger; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.TopicConfig; +import com.alibaba.rocketmq.common.help.FAQUrl; +import com.alibaba.rocketmq.common.message.MessageConst; +import com.alibaba.rocketmq.common.message.MessageDecoder; +import com.alibaba.rocketmq.common.message.MessageExt; +import com.alibaba.rocketmq.common.message.MessageId; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.common.protocol.ResponseCode; +import com.alibaba.rocketmq.common.protocol.header.QueryMessageRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.QueryMessageResponseHeader; +import com.alibaba.rocketmq.common.protocol.route.BrokerData; +import com.alibaba.rocketmq.common.protocol.route.TopicRouteData; +import com.alibaba.rocketmq.remoting.InvokeCallback; +import com.alibaba.rocketmq.remoting.common.RemotingUtil; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; +import com.alibaba.rocketmq.remoting.exception.RemotingException; +import com.alibaba.rocketmq.remoting.netty.ResponseFuture; +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; + + +/** + * 管理类接口实现 + * + * @author shijia.wxr + * @since 2013-7-24 + */ +public class MQAdminImpl { + private final Logger log = ClientLogger.getLog(); + private final MQClientInstance mQClientFactory; + + + public MQAdminImpl(MQClientInstance mQClientFactory) { + this.mQClientFactory = mQClientFactory; + } + + + public void createTopic(String key, String newTopic, int queueNum) throws MQClientException { + createTopic(key, newTopic, queueNum, 0); + } + + + public void createTopic(String key, String newTopic, int queueNum, int topicSysFlag) + throws MQClientException { + try { + TopicRouteData topicRouteData = + this.mQClientFactory.getMQClientAPIImpl().getTopicRouteInfoFromNameServer(key, 1000 * 3); + List brokerDataList = topicRouteData.getBrokerDatas(); + if (brokerDataList != null && !brokerDataList.isEmpty()) { + // 排序原因：即使没有配置顺序消息模式，默认队列的顺序同配置的一致。 + Collections.sort(brokerDataList); + + MQClientException exception = null; + + StringBuilder orderTopicString = new StringBuilder(); + + // 遍历各个Broker + for (BrokerData brokerData : brokerDataList) { + String addr = brokerData.getBrokerAddrs().get(MixAll.MASTER_ID); + if (addr != null) { + TopicConfig topicConfig = new TopicConfig(newTopic); + topicConfig.setReadQueueNums(queueNum); + topicConfig.setWriteQueueNums(queueNum); + topicConfig.setTopicSysFlag(topicSysFlag); + try { + this.mQClientFactory.getMQClientAPIImpl().createTopic(addr, key, topicConfig, + 1000 * 3); + } + catch (Exception e) { + exception = new MQClientException("create topic to broker exception", e); + } + + orderTopicString.append(brokerData.getBrokerName()); + orderTopicString.append(":"); + orderTopicString.append(queueNum); + orderTopicString.append(";"); + } + } + + if (exception != null) { + throw exception; + } + } + else { + throw new MQClientException("Not found broker, maybe key is wrong", null); + } + } + catch (Exception e) { + throw new MQClientException("create new topic failed", e); + } + } + + + public List fetchPublishMessageQueues(String topic) throws MQClientException { + try { + TopicRouteData topicRouteData = + this.mQClientFactory.getMQClientAPIImpl() + .getTopicRouteInfoFromNameServer(topic, 1000 * 3); + if (topicRouteData != null) { + TopicPublishInfo topicPublishInfo = + MQClientInstance.topicRouteData2TopicPublishInfo(topic, topicRouteData); + if (topicPublishInfo != null && topicPublishInfo.ok()) { + return topicPublishInfo.getMessageQueueList(); + } + } + } + catch (Exception e) { + throw new MQClientException("Can not find Message Queue for this topic, " + topic, e); + } + + throw new MQClientException("Unknow why, Can not find Message Queue for this topic, " + topic, null); + } + + + public Set fetchSubscribeMessageQueues(String topic) throws MQClientException { + try { + TopicRouteData topicRouteData = + this.mQClientFactory.getMQClientAPIImpl() + .getTopicRouteInfoFromNameServer(topic, 1000 * 3); + if (topicRouteData != null) { + Set mqList = + MQClientInstance.topicRouteData2TopicSubscribeInfo(topic, topicRouteData); + if (!mqList.isEmpty()) { + return mqList; + } + else { + throw new MQClientException("Can not find Message Queue for this topic, " + topic + + " Namesrv return empty", null); + } + } + } + catch (Exception e) { + throw new MQClientException("Can not find Message Queue for this topic, " + topic + + FAQUrl.suggestTodo(FAQUrl.MQLIST_NOT_EXIST), // + e); + } + + throw new MQClientException("Unknow why, Can not find Message Queue for this topic, " + topic, null); + } + + + public long searchOffset(MessageQueue mq, long timestamp) throws MQClientException { + String brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(mq.getBrokerName()); + if (null == brokerAddr) { + this.mQClientFactory.updateTopicRouteInfoFromNameServer(mq.getTopic()); + brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(mq.getBrokerName()); + } + + if (brokerAddr != null) { + try { + return this.mQClientFactory.getMQClientAPIImpl().searchOffset(brokerAddr, mq.getTopic(), + mq.getQueueId(), timestamp, 1000 * 3); + } + catch (Exception e) { + throw new MQClientException("Invoke Broker[" + brokerAddr + "] exception", e); + } + } + + throw new MQClientException("The broker[" + mq.getBrokerName() + "] not exist", null); + } + + + public long maxOffset(MessageQueue mq) throws MQClientException { + String brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(mq.getBrokerName()); + if (null == brokerAddr) { + this.mQClientFactory.updateTopicRouteInfoFromNameServer(mq.getTopic()); + brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(mq.getBrokerName()); + } + + if (brokerAddr != null) { + try { + return this.mQClientFactory.getMQClientAPIImpl().getMaxOffset(brokerAddr, mq.getTopic(), + mq.getQueueId(), 1000 * 3); + } + catch (Exception e) { + throw new MQClientException("Invoke Broker[" + brokerAddr + "] exception", e); + } + } + + throw new MQClientException("The broker[" + mq.getBrokerName() + "] not exist", null); + } + + + public long minOffset(MessageQueue mq) throws MQClientException { + String brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(mq.getBrokerName()); + if (null == brokerAddr) { + this.mQClientFactory.updateTopicRouteInfoFromNameServer(mq.getTopic()); + brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(mq.getBrokerName()); + } + + if (brokerAddr != null) { + try { + return this.mQClientFactory.getMQClientAPIImpl().getMinOffset(brokerAddr, mq.getTopic(), + mq.getQueueId(), 1000 * 3); + } + catch (Exception e) { + throw new MQClientException("Invoke Broker[" + brokerAddr + "] exception", e); + } + } + + throw new MQClientException("The broker[" + mq.getBrokerName() + "] not exist", null); + } + + + public long earliestMsgStoreTime(MessageQueue mq) throws MQClientException { + String brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(mq.getBrokerName()); + if (null == brokerAddr) { + this.mQClientFactory.updateTopicRouteInfoFromNameServer(mq.getTopic()); + brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(mq.getBrokerName()); + } + + if (brokerAddr != null) { + try { + return this.mQClientFactory.getMQClientAPIImpl().getEarliestMsgStoretime(brokerAddr, + mq.getTopic(), mq.getQueueId(), 1000 * 3); + } + catch (Exception e) { + throw new MQClientException("Invoke Broker[" + brokerAddr + "] exception", e); + } + } + + throw new MQClientException("The broker[" + mq.getBrokerName() + "] not exist", null); + } + + + public MessageExt viewMessage(String msgId) throws RemotingException, MQBrokerException, + InterruptedException, MQClientException { + try { + MessageId messageId = MessageDecoder.decodeMessageId(msgId); + return this.mQClientFactory.getMQClientAPIImpl().viewMessage( + RemotingUtil.socketAddress2String(messageId.getAddress()), messageId.getOffset(), 1000 * 3); + } + catch (UnknownHostException e) { + throw new MQClientException("message id illegal", e); + } + } + + + public QueryResult queryMessage(String topic, String key, int maxNum, long begin, long end) + throws MQClientException, InterruptedException { + TopicRouteData topicRouteData = this.mQClientFactory.getAnExistTopicRouteData(topic); + if (null == topicRouteData) { + this.mQClientFactory.updateTopicRouteInfoFromNameServer(topic); + topicRouteData = this.mQClientFactory.getAnExistTopicRouteData(topic); + } + + if (topicRouteData != null) { + List brokerAddrs = new LinkedList(); + for (BrokerData brokerData : topicRouteData.getBrokerDatas()) { + String addr = brokerData.selectBrokerAddr(); + if (addr != null) { + brokerAddrs.add(addr); + } + } + + if (!brokerAddrs.isEmpty()) { + final CountDownLatch countDownLatch = new CountDownLatch(brokerAddrs.size()); + final List queryResultList = new LinkedList(); + + for (String addr : brokerAddrs) { + try { + QueryMessageRequestHeader requestHeader = new QueryMessageRequestHeader(); + requestHeader.setTopic(topic); + requestHeader.setKey(key); + requestHeader.setMaxNum(maxNum); + requestHeader.setBeginTimestamp(begin); + requestHeader.setEndTimestamp(end); + + this.mQClientFactory.getMQClientAPIImpl().queryMessage(addr, requestHeader, + 1000 * 15, new InvokeCallback() { + @Override + public void operationComplete(ResponseFuture responseFuture) { + try { + RemotingCommand response = responseFuture.getResponseCommand(); + if (response != null) { + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + QueryMessageResponseHeader responseHeader = null; + try { + responseHeader = + (QueryMessageResponseHeader) response + .decodeCommandCustomHeader(QueryMessageResponseHeader.class); + } + catch (RemotingCommandException e) { + log.error("decodeCommandCustomHeader exception", e); + return; + } + + List wrappers = + MessageDecoder.decodes( + ByteBuffer.wrap(response.getBody()), true); + + QueryResult qr = + new QueryResult(responseHeader + .getIndexLastUpdateTimestamp(), wrappers); + queryResultList.add(qr); + break; + } + default: + log.warn("getResponseCommand failed, {} {}", + response.getCode(), response.getRemark()); + break; + } + } + else { + log.warn("getResponseCommand return null"); + } + } + finally { + countDownLatch.countDown(); + } + } + }); + } + catch (Exception e) { + log.warn("queryMessage exception", e); + }// end of try + + } // end of for + + boolean ok = countDownLatch.await(1000 * 20, TimeUnit.MILLISECONDS); + if (!ok) { + log.warn("queryMessage, maybe some broker failed"); + } + + long indexLastUpdateTimestamp = 0; + List messageList = new LinkedList(); + for (QueryResult qr : queryResultList) { + if (qr.getIndexLastUpdateTimestamp() > indexLastUpdateTimestamp) { + indexLastUpdateTimestamp = qr.getIndexLastUpdateTimestamp(); + } + + for (MessageExt msgExt : qr.getMessageList()) { + String keys = msgExt.getKeys(); + if (keys != null) { + boolean matched = false; + String[] keyArray = keys.split(MessageConst.KEY_SEPARATOR); + if (keyArray != null) { + for (String k : keyArray) { + if (key.equals(k)) { + matched = true; + break; + } + } + } + + if (matched) { + messageList.add(msgExt); + } + else { + log.warn( + "queryMessage, find message key not matched, maybe hash duplicate {}", + msgExt.toString()); + } + } + } + } + + if (!messageList.isEmpty()) { + return new QueryResult(indexLastUpdateTimestamp, messageList); + } + else { + throw new MQClientException("query operation over, but no message.", null); + } + } + } + + throw new MQClientException("The topic[" + topic + "] not matched route info", null); + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/MQClientAPIImpl.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/MQClientAPIImpl.java new file mode 100644 index 000000000..4d4ab629e --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/MQClientAPIImpl.java @@ -0,0 +1,2407 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.impl; + +import java.io.UnsupportedEncodingException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +import org.slf4j.Logger; + +import com.alibaba.rocketmq.client.VirtualEnvUtil; +import com.alibaba.rocketmq.client.consumer.PullCallback; +import com.alibaba.rocketmq.client.consumer.PullResult; +import com.alibaba.rocketmq.client.consumer.PullStatus; +import com.alibaba.rocketmq.client.exception.MQBrokerException; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.client.impl.consumer.PullResultExt; +import com.alibaba.rocketmq.client.log.ClientLogger; +import com.alibaba.rocketmq.client.producer.SendCallback; +import com.alibaba.rocketmq.client.producer.SendResult; +import com.alibaba.rocketmq.client.producer.SendStatus; +import com.alibaba.rocketmq.common.MQVersion; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.TopicConfig; +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.admin.ConsumeStats; +import com.alibaba.rocketmq.common.admin.OffsetWrapper; +import com.alibaba.rocketmq.common.admin.TopicOffset; +import com.alibaba.rocketmq.common.admin.TopicStatsTable; +import com.alibaba.rocketmq.common.message.Message; +import com.alibaba.rocketmq.common.message.MessageDecoder; +import com.alibaba.rocketmq.common.message.MessageExt; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.common.namesrv.NamesrvUtil; +import com.alibaba.rocketmq.common.namesrv.TopAddressing; +import com.alibaba.rocketmq.common.protocol.RequestCode; +import com.alibaba.rocketmq.common.protocol.ResponseCode; +import com.alibaba.rocketmq.common.protocol.body.ClusterInfo; +import com.alibaba.rocketmq.common.protocol.body.ConsumeMessageDirectlyResult; +import com.alibaba.rocketmq.common.protocol.body.ConsumerConnection; +import com.alibaba.rocketmq.common.protocol.body.ConsumerRunningInfo; +import com.alibaba.rocketmq.common.protocol.body.GetConsumerStatusBody; +import com.alibaba.rocketmq.common.protocol.body.GroupList; +import com.alibaba.rocketmq.common.protocol.body.KVTable; +import com.alibaba.rocketmq.common.protocol.body.LockBatchRequestBody; +import com.alibaba.rocketmq.common.protocol.body.LockBatchResponseBody; +import com.alibaba.rocketmq.common.protocol.body.ProducerConnection; +import com.alibaba.rocketmq.common.protocol.body.QueryConsumeTimeSpanBody; +import com.alibaba.rocketmq.common.protocol.body.QueryCorrectionOffsetBody; +import com.alibaba.rocketmq.common.protocol.body.QueueTimeSpan; +import com.alibaba.rocketmq.common.protocol.body.ResetOffsetBody; +import com.alibaba.rocketmq.common.protocol.body.TopicList; +import com.alibaba.rocketmq.common.protocol.body.UnlockBatchRequestBody; +import com.alibaba.rocketmq.common.protocol.header.CloneGroupOffsetRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.ConsumeMessageDirectlyResultRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.ConsumerSendMsgBackRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.CreateTopicRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.DeleteSubscriptionGroupRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.DeleteTopicRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.EndTransactionRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.GetConsumeStatsRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.GetConsumerConnectionListRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.GetConsumerListByGroupRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.GetConsumerListByGroupResponseBody; +import com.alibaba.rocketmq.common.protocol.header.GetConsumerRunningInfoRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.GetConsumerStatusRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.GetEarliestMsgStoretimeRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.GetEarliestMsgStoretimeResponseHeader; +import com.alibaba.rocketmq.common.protocol.header.GetMaxOffsetRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.GetMaxOffsetResponseHeader; +import com.alibaba.rocketmq.common.protocol.header.GetMinOffsetRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.GetMinOffsetResponseHeader; +import com.alibaba.rocketmq.common.protocol.header.GetProducerConnectionListRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.GetTopicStatsInfoRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.GetTopicsByClusterRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.PullMessageRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.PullMessageResponseHeader; +import com.alibaba.rocketmq.common.protocol.header.QueryConsumeTimeSpanRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.QueryConsumerOffsetRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.QueryConsumerOffsetResponseHeader; +import com.alibaba.rocketmq.common.protocol.header.QueryCorrectionOffsetHeader; +import com.alibaba.rocketmq.common.protocol.header.QueryMessageRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.QueryTopicConsumeByWhoRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.ResetOffsetRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.SearchOffsetRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.SearchOffsetResponseHeader; +import com.alibaba.rocketmq.common.protocol.header.SendMessageRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.SendMessageRequestHeaderV2; +import com.alibaba.rocketmq.common.protocol.header.SendMessageResponseHeader; +import com.alibaba.rocketmq.common.protocol.header.UnregisterClientRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.UpdateConsumerOffsetRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.ViewMessageRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.filtersrv.RegisterMessageFilterClassRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.namesrv.DeleteKVConfigRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.namesrv.GetKVConfigRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.namesrv.GetKVConfigResponseHeader; +import com.alibaba.rocketmq.common.protocol.header.namesrv.GetKVListByNamespaceRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.namesrv.GetRouteInfoRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.namesrv.PutKVConfigRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.namesrv.WipeWritePermOfBrokerRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.namesrv.WipeWritePermOfBrokerResponseHeader; +import com.alibaba.rocketmq.common.protocol.heartbeat.ConsumerData; +import com.alibaba.rocketmq.common.protocol.heartbeat.HeartbeatData; +import com.alibaba.rocketmq.common.protocol.heartbeat.ProducerData; +import com.alibaba.rocketmq.common.protocol.heartbeat.SubscriptionData; +import com.alibaba.rocketmq.common.protocol.route.TopicRouteData; +import com.alibaba.rocketmq.common.subscription.SubscriptionGroupConfig; +import com.alibaba.rocketmq.remoting.InvokeCallback; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.remoting.RemotingClient; +import com.alibaba.rocketmq.remoting.common.RemotingUtil; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; +import com.alibaba.rocketmq.remoting.exception.RemotingConnectException; +import com.alibaba.rocketmq.remoting.exception.RemotingException; +import com.alibaba.rocketmq.remoting.exception.RemotingSendRequestException; +import com.alibaba.rocketmq.remoting.exception.RemotingTimeoutException; +import com.alibaba.rocketmq.remoting.exception.RemotingTooMuchRequestException; +import com.alibaba.rocketmq.remoting.netty.NettyClientConfig; +import com.alibaba.rocketmq.remoting.netty.NettyRemotingClient; +import com.alibaba.rocketmq.remoting.netty.ResponseFuture; +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; + + +/** + * 封装所有与服务器通信部分API + * + * @author shijia.wxr + * @since 2013-7-24 + */ +public class MQClientAPIImpl { + + static { + System.setProperty(RemotingCommand.RemotingVersionKey, Integer.toString(MQVersion.CurrentVersion)); + } + + private final static Logger log = ClientLogger.getLog(); + private final RemotingClient remotingClient; + private final TopAddressing topAddressing = new TopAddressing(MixAll.WS_ADDR); + private final ClientRemotingProcessor clientRemotingProcessor; + private String nameSrvAddr = null; + // 虚拟运行环境相关的project group + private String projectGroupPrefix; + + + public MQClientAPIImpl(final NettyClientConfig nettyClientConfig, + final ClientRemotingProcessor clientRemotingProcessor, RPCHook rpcHook) { + this.remotingClient = new NettyRemotingClient(nettyClientConfig, null); + this.clientRemotingProcessor = clientRemotingProcessor; + + this.remotingClient.registerRPCHook(rpcHook); + /** + * 注册客户端支持的RPC CODE + */ + this.remotingClient.registerProcessor(RequestCode.CHECK_TRANSACTION_STATE, + this.clientRemotingProcessor, null); + + this.remotingClient.registerProcessor(RequestCode.NOTIFY_CONSUMER_IDS_CHANGED, + this.clientRemotingProcessor, null); + + this.remotingClient.registerProcessor(RequestCode.RESET_CONSUMER_CLIENT_OFFSET, + this.clientRemotingProcessor, null); + + this.remotingClient.registerProcessor(RequestCode.GET_CONSUMER_STATUS_FROM_CLIENT, + this.clientRemotingProcessor, null); + + this.remotingClient.registerProcessor(RequestCode.GET_CONSUMER_RUNNING_INFO, + this.clientRemotingProcessor, null); + + this.remotingClient.registerProcessor(RequestCode.CONSUME_MESSAGE_DIRECTLY, + this.clientRemotingProcessor, null); + } + + + public MQClientAPIImpl(final NettyClientConfig nettyClientConfig, + final ClientRemotingProcessor clientRemotingProcessor) { + this(nettyClientConfig, clientRemotingProcessor, null); + } + + + public List getNameServerAddressList() { + return this.remotingClient.getNameServerAddressList(); + } + + + public RemotingClient getRemotingClient() { + return remotingClient; + } + + + public String fetchNameServerAddr() { + try { + String addrs = this.topAddressing.fetchNSAddr(); + if (addrs != null) { + if (!addrs.equals(this.nameSrvAddr)) { + log.info("name server address changed, old: " + this.nameSrvAddr + " new: " + addrs); + this.updateNameServerAddressList(addrs); + this.nameSrvAddr = addrs; + return nameSrvAddr; + } + } + } + catch (Exception e) { + log.error("fetchNameServerAddr Exception", e); + } + return nameSrvAddr; + } + + + public void updateNameServerAddressList(final String addrs) { + List lst = new ArrayList(); + String[] addrArray = addrs.split(";"); + if (addrArray != null) { + for (String addr : addrArray) { + lst.add(addr); + } + + this.remotingClient.updateNameServerAddressList(lst); + } + } + + + public void start() { + // 远程通信 Client 启动 + this.remotingClient.start(); + + // 获取虚拟运行环境相关的project group + try { + String localAddress = RemotingUtil.getLocalAddress(); + projectGroupPrefix = this.getProjectGroupByIp(localAddress, 3000); + log.info("The client[{}] in project group: {}", localAddress, projectGroupPrefix); + } + catch (Exception e) { + } + } + + + public void shutdown() { + this.remotingClient.shutdown(); + } + + + public void createSubscriptionGroup(final String addr, final SubscriptionGroupConfig config, + final long timeoutMillis) throws RemotingException, MQBrokerException, InterruptedException, + MQClientException { + // 添加虚拟运行环境相关的projectGroupPrefix + if (!UtilAll.isBlank(projectGroupPrefix)) { + config.setGroupName(VirtualEnvUtil.buildWithProjectGroup(config.getGroupName(), + projectGroupPrefix)); + } + + RemotingCommand request = + RemotingCommand.createRequestCommand(RequestCode.UPDATE_AND_CREATE_SUBSCRIPTIONGROUP, null); + + byte[] body = RemotingSerializable.encode(config); + request.setBody(body); + + RemotingCommand response = this.remotingClient.invokeSync(addr, request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + return; + } + default: + break; + } + + throw new MQClientException(response.getCode(), response.getRemark()); + + } + + + public void createTopic(final String addr, final String defaultTopic, final TopicConfig topicConfig, + final long timeoutMillis) throws RemotingException, MQBrokerException, InterruptedException, + MQClientException { + // 添加虚拟运行环境相关的projectGroupPrefix + String topicWithProjectGroup = topicConfig.getTopicName(); + if (!UtilAll.isBlank(projectGroupPrefix)) { + topicWithProjectGroup = + VirtualEnvUtil.buildWithProjectGroup(topicConfig.getTopicName(), projectGroupPrefix); + } + + CreateTopicRequestHeader requestHeader = new CreateTopicRequestHeader(); + requestHeader.setTopic(topicWithProjectGroup); + requestHeader.setDefaultTopic(defaultTopic); + requestHeader.setReadQueueNums(topicConfig.getReadQueueNums()); + requestHeader.setWriteQueueNums(topicConfig.getWriteQueueNums()); + requestHeader.setPerm(topicConfig.getPerm()); + requestHeader.setTopicFilterType(topicConfig.getTopicFilterType().name()); + requestHeader.setTopicSysFlag(topicConfig.getTopicSysFlag()); + requestHeader.setOrder(topicConfig.isOrder()); + + RemotingCommand request = + RemotingCommand.createRequestCommand(RequestCode.UPDATE_AND_CREATE_TOPIC, requestHeader); + + RemotingCommand response = this.remotingClient.invokeSync(addr, request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + return; + } + default: + break; + } + + throw new MQClientException(response.getCode(), response.getRemark()); + } + + /** + * 是否发送网络包精简的Message + */ + public static boolean sendSmartMsg = // + Boolean.parseBoolean(System.getProperty("com.alibaba.rocketmq.client.sendSmartMsg", "true")); + + + /** + * 发送消息 + */ + public SendResult sendMessage(// + final String addr,// 1 + final String brokerName,// 2 + final Message msg,// 3 + final SendMessageRequestHeader requestHeader,// 4 + final long timeoutMillis,// 5 + final CommunicationMode communicationMode,// 6 + final SendCallback sendCallback// 7 + ) throws RemotingException, MQBrokerException, InterruptedException { + // 添加虚拟运行环境相关的projectGroupPrefix + if (!UtilAll.isBlank(projectGroupPrefix)) { + msg.setTopic(VirtualEnvUtil.buildWithProjectGroup(msg.getTopic(), projectGroupPrefix)); + requestHeader.setProducerGroup(VirtualEnvUtil.buildWithProjectGroup( + requestHeader.getProducerGroup(), projectGroupPrefix)); + requestHeader.setTopic(VirtualEnvUtil.buildWithProjectGroup(requestHeader.getTopic(), + projectGroupPrefix)); + } + + RemotingCommand request = null; + if (sendSmartMsg) { + SendMessageRequestHeaderV2 requestHeaderV2 = + SendMessageRequestHeaderV2.createSendMessageRequestHeaderV2(requestHeader); + request = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE_V2, requestHeaderV2); + } + else { + request = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE, requestHeader); + } + + request.setBody(msg.getBody()); + + switch (communicationMode) { + case ONEWAY: + this.remotingClient.invokeOneway(addr, request, timeoutMillis); + return null; + case ASYNC: + this.sendMessageAsync(addr, brokerName, msg, timeoutMillis, request, sendCallback); + return null; + case SYNC: + return this.sendMessageSync(addr, brokerName, msg, timeoutMillis, request); + default: + assert false; + break; + } + + return null; + } + + + private SendResult sendMessageSync(// + final String addr,// + final String brokerName,// + final Message msg,// + final long timeoutMillis,// + final RemotingCommand request// + ) throws RemotingException, MQBrokerException, InterruptedException { + RemotingCommand response = this.remotingClient.invokeSync(addr, request, timeoutMillis); + assert response != null; + return this.processSendResponse(brokerName, msg, response); + } + + + private void sendMessageAsync(// + final String addr,// + final String brokerName,// + final Message msg,// + final long timeoutMillis,// + final RemotingCommand request,// + final SendCallback sendCallback// + ) throws RemotingException, InterruptedException { + this.remotingClient.invokeAsync(addr, request, timeoutMillis, new InvokeCallback() { + @Override + public void operationComplete(ResponseFuture responseFuture) { + if (null == sendCallback) + return; + + RemotingCommand response = responseFuture.getResponseCommand(); + if (response != null) { + try { + SendResult sendResult = + MQClientAPIImpl.this.processSendResponse(brokerName, msg, response); + assert sendResult != null; + sendCallback.onSuccess(sendResult); + } + catch (Exception e) { + sendCallback.onException(e); + } + } + else { + if (!responseFuture.isSendRequestOK()) { + sendCallback.onException(new MQClientException("send request failed", responseFuture + .getCause())); + } + else if (responseFuture.isTimeout()) { + sendCallback.onException(new MQClientException("wait response timeout " + + responseFuture.getTimeoutMillis() + "ms", responseFuture.getCause())); + } + else { + sendCallback.onException(new MQClientException("unknow reseaon", responseFuture + .getCause())); + } + } + } + }); + } + + + private SendResult processSendResponse(// + final String brokerName,// + final Message msg,// + final RemotingCommand response// + ) throws MQBrokerException, RemotingCommandException { + switch (response.getCode()) { + case ResponseCode.FLUSH_DISK_TIMEOUT: + case ResponseCode.FLUSH_SLAVE_TIMEOUT: + case ResponseCode.SLAVE_NOT_AVAILABLE: { + // TODO LOG + } + case ResponseCode.SUCCESS: { + SendStatus sendStatus = SendStatus.SEND_OK; + switch (response.getCode()) { + case ResponseCode.FLUSH_DISK_TIMEOUT: + sendStatus = SendStatus.FLUSH_DISK_TIMEOUT; + break; + case ResponseCode.FLUSH_SLAVE_TIMEOUT: + sendStatus = SendStatus.FLUSH_SLAVE_TIMEOUT; + break; + case ResponseCode.SLAVE_NOT_AVAILABLE: + sendStatus = SendStatus.SLAVE_NOT_AVAILABLE; + break; + case ResponseCode.SUCCESS: + sendStatus = SendStatus.SEND_OK; + break; + default: + assert false; + break; + } + + SendMessageResponseHeader responseHeader = + (SendMessageResponseHeader) response + .decodeCommandCustomHeader(SendMessageResponseHeader.class); + + MessageQueue messageQueue = + new MessageQueue(msg.getTopic(), brokerName, responseHeader.getQueueId()); + + return new SendResult(sendStatus, responseHeader.getMsgId(), messageQueue, + responseHeader.getQueueOffset(), projectGroupPrefix); + } + default: + break; + } + + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + + + /** + * 拉消息接口 + */ + public PullResult pullMessage(// + final String addr,// + final PullMessageRequestHeader requestHeader,// + final long timeoutMillis,// + final CommunicationMode communicationMode,// + final PullCallback pullCallback// + ) throws RemotingException, MQBrokerException, InterruptedException { + // 添加虚拟运行环境相关的projectGroupPrefix + if (!UtilAll.isBlank(projectGroupPrefix)) { + requestHeader.setConsumerGroup(VirtualEnvUtil.buildWithProjectGroup( + requestHeader.getConsumerGroup(), projectGroupPrefix)); + requestHeader.setTopic(VirtualEnvUtil.buildWithProjectGroup(requestHeader.getTopic(), + projectGroupPrefix)); + } + + RemotingCommand request = + RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, requestHeader); + + switch (communicationMode) { + case ONEWAY: + assert false; + return null; + case ASYNC: + this.pullMessageAsync(addr, request, timeoutMillis, pullCallback); + return null; + case SYNC: + return this.pullMessageSync(addr, request, timeoutMillis); + default: + assert false; + break; + } + + return null; + } + + + private void pullMessageAsync(// + final String addr,// 1 + final RemotingCommand request,// + final long timeoutMillis,// + final PullCallback pullCallback// + ) throws RemotingException, InterruptedException { + this.remotingClient.invokeAsync(addr, request, timeoutMillis, new InvokeCallback() { + @Override + public void operationComplete(ResponseFuture responseFuture) { + RemotingCommand response = responseFuture.getResponseCommand(); + if (response != null) { + try { + PullResult pullResult = MQClientAPIImpl.this.processPullResponse(response); + assert pullResult != null; + pullCallback.onSuccess(pullResult); + } + catch (Exception e) { + pullCallback.onException(e); + } + } + else { + if (!responseFuture.isSendRequestOK()) { + pullCallback.onException(new MQClientException("send request failed", responseFuture + .getCause())); + } + else if (responseFuture.isTimeout()) { + pullCallback.onException(new MQClientException("wait response timeout " + + responseFuture.getTimeoutMillis() + "ms", responseFuture.getCause())); + } + else { + pullCallback.onException(new MQClientException("unknow reseaon", responseFuture + .getCause())); + } + } + } + }); + } + + + private PullResult processPullResponse(final RemotingCommand response) throws MQBrokerException, + RemotingCommandException { + PullStatus pullStatus = PullStatus.NO_NEW_MSG; + switch (response.getCode()) { + case ResponseCode.SUCCESS: + pullStatus = PullStatus.FOUND; + break; + case ResponseCode.PULL_NOT_FOUND: + pullStatus = PullStatus.NO_NEW_MSG; + break; + case ResponseCode.PULL_RETRY_IMMEDIATELY: + pullStatus = PullStatus.NO_MATCHED_MSG; + break; + case ResponseCode.PULL_OFFSET_MOVED: + pullStatus = PullStatus.OFFSET_ILLEGAL; + break; + + default: + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + + PullMessageResponseHeader responseHeader = + (PullMessageResponseHeader) response + .decodeCommandCustomHeader(PullMessageResponseHeader.class); + + return new PullResultExt(pullStatus, responseHeader.getNextBeginOffset(), + responseHeader.getMinOffset(), responseHeader.getMaxOffset(), null, + responseHeader.getSuggestWhichBrokerId(), response.getBody()); + } + + + private PullResult pullMessageSync(// + final String addr,// 1 + final RemotingCommand request,// 2 + final long timeoutMillis// 3 + ) throws RemotingException, InterruptedException, MQBrokerException { + RemotingCommand response = this.remotingClient.invokeSync(addr, request, timeoutMillis); + assert response != null; + return this.processPullResponse(response); + } + + + /** + * 根据时间查询Offset + */ + public MessageExt viewMessage(final String addr, final long phyoffset, final long timeoutMillis) + throws RemotingException, MQBrokerException, InterruptedException { + ViewMessageRequestHeader requestHeader = new ViewMessageRequestHeader(); + requestHeader.setOffset(phyoffset); + RemotingCommand request = + RemotingCommand.createRequestCommand(RequestCode.VIEW_MESSAGE_BY_ID, requestHeader); + + RemotingCommand response = this.remotingClient.invokeSync(addr, request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + ByteBuffer byteBuffer = ByteBuffer.wrap(response.getBody()); + MessageExt messageExt = MessageDecoder.decode(byteBuffer); + // 清除虚拟运行环境相关的projectGroupPrefix + if (!UtilAll.isBlank(projectGroupPrefix)) { + messageExt.setTopic(VirtualEnvUtil.clearProjectGroup(messageExt.getTopic(), + projectGroupPrefix)); + } + return messageExt; + } + default: + break; + } + + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + + + /** + * 根据时间查询Offset + */ + public long searchOffset(final String addr, final String topic, final int queueId, final long timestamp, + final long timeoutMillis) throws RemotingException, MQBrokerException, InterruptedException { + // 添加虚拟运行环境相关的projectGroupPrefix + String topicWithProjectGroup = topic; + if (!UtilAll.isBlank(projectGroupPrefix)) { + topicWithProjectGroup = VirtualEnvUtil.buildWithProjectGroup(topic, projectGroupPrefix); + } + + SearchOffsetRequestHeader requestHeader = new SearchOffsetRequestHeader(); + requestHeader.setTopic(topicWithProjectGroup); + requestHeader.setQueueId(queueId); + requestHeader.setTimestamp(timestamp); + RemotingCommand request = + RemotingCommand.createRequestCommand(RequestCode.SEARCH_OFFSET_BY_TIMESTAMP, requestHeader); + + RemotingCommand response = this.remotingClient.invokeSync(addr, request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + SearchOffsetResponseHeader responseHeader = + (SearchOffsetResponseHeader) response + .decodeCommandCustomHeader(SearchOffsetResponseHeader.class); + return responseHeader.getOffset(); + } + default: + break; + } + + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + + + /** + * 获取队列的最大Offset + */ + public long getMaxOffset(final String addr, final String topic, final int queueId, + final long timeoutMillis) throws RemotingException, MQBrokerException, InterruptedException { + // 添加虚拟运行环境相关的projectGroupPrefix + String topicWithProjectGroup = topic; + if (!UtilAll.isBlank(projectGroupPrefix)) { + topicWithProjectGroup = VirtualEnvUtil.buildWithProjectGroup(topic, projectGroupPrefix); + } + + GetMaxOffsetRequestHeader requestHeader = new GetMaxOffsetRequestHeader(); + requestHeader.setTopic(topicWithProjectGroup); + requestHeader.setQueueId(queueId); + RemotingCommand request = + RemotingCommand.createRequestCommand(RequestCode.GET_MAX_OFFSET, requestHeader); + + RemotingCommand response = this.remotingClient.invokeSync(addr, request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + GetMaxOffsetResponseHeader responseHeader = + (GetMaxOffsetResponseHeader) response + .decodeCommandCustomHeader(GetMaxOffsetResponseHeader.class); + + return responseHeader.getOffset(); + } + default: + break; + } + + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + + + /** + * 获取某个组的Consumer Id列表 + */ + public List getConsumerIdListByGroup(// + final String addr, // + final String consumerGroup, // + final long timeoutMillis) throws RemotingConnectException, RemotingSendRequestException, + RemotingTimeoutException, MQBrokerException, InterruptedException { + // 添加虚拟运行环境相关的projectGroupPrefix + String consumerGroupWithProjectGroup = consumerGroup; + if (!UtilAll.isBlank(projectGroupPrefix)) { + consumerGroupWithProjectGroup = + VirtualEnvUtil.buildWithProjectGroup(consumerGroup, projectGroupPrefix); + } + + GetConsumerListByGroupRequestHeader requestHeader = new GetConsumerListByGroupRequestHeader(); + requestHeader.setConsumerGroup(consumerGroupWithProjectGroup); + RemotingCommand request = + RemotingCommand.createRequestCommand(RequestCode.GET_CONSUMER_LIST_BY_GROUP, requestHeader); + + RemotingCommand response = this.remotingClient.invokeSync(addr, request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + if (response.getBody() != null) { + GetConsumerListByGroupResponseBody body = + GetConsumerListByGroupResponseBody.decode(response.getBody(), + GetConsumerListByGroupResponseBody.class); + return body.getConsumerIdList(); + } + } + default: + break; + } + + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + + + /** + * 获取队列的最小Offset + */ + public long getMinOffset(final String addr, final String topic, final int queueId, + final long timeoutMillis) throws RemotingException, MQBrokerException, InterruptedException { + // 添加虚拟运行环境相关的projectGroupPrefix + String topicWithProjectGroup = topic; + if (!UtilAll.isBlank(projectGroupPrefix)) { + topicWithProjectGroup = VirtualEnvUtil.buildWithProjectGroup(topic, projectGroupPrefix); + } + + GetMinOffsetRequestHeader requestHeader = new GetMinOffsetRequestHeader(); + requestHeader.setTopic(topicWithProjectGroup); + requestHeader.setQueueId(queueId); + RemotingCommand request = + RemotingCommand.createRequestCommand(RequestCode.GET_MIN_OFFSET, requestHeader); + + RemotingCommand response = this.remotingClient.invokeSync(addr, request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + GetMinOffsetResponseHeader responseHeader = + (GetMinOffsetResponseHeader) response + .decodeCommandCustomHeader(GetMinOffsetResponseHeader.class); + + return responseHeader.getOffset(); + } + default: + break; + } + + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + + + /** + * 获取队列的最早时间 + */ + public long getEarliestMsgStoretime(final String addr, final String topic, final int queueId, + final long timeoutMillis) throws RemotingException, MQBrokerException, InterruptedException { + // 添加虚拟运行环境相关的projectGroupPrefix + String topicWithProjectGroup = topic; + if (!UtilAll.isBlank(projectGroupPrefix)) { + topicWithProjectGroup = VirtualEnvUtil.buildWithProjectGroup(topic, projectGroupPrefix); + } + + GetEarliestMsgStoretimeRequestHeader requestHeader = new GetEarliestMsgStoretimeRequestHeader(); + requestHeader.setTopic(topicWithProjectGroup); + requestHeader.setQueueId(queueId); + RemotingCommand request = + RemotingCommand.createRequestCommand(RequestCode.GET_EARLIEST_MSG_STORETIME, requestHeader); + + RemotingCommand response = this.remotingClient.invokeSync(addr, request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + GetEarliestMsgStoretimeResponseHeader responseHeader = + (GetEarliestMsgStoretimeResponseHeader) response + .decodeCommandCustomHeader(GetEarliestMsgStoretimeResponseHeader.class); + + return responseHeader.getTimestamp(); + } + default: + break; + } + + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + + + /** + * 查询Consumer消费进度 + */ + public long queryConsumerOffset(// + final String addr,// + final QueryConsumerOffsetRequestHeader requestHeader,// + final long timeoutMillis// + ) throws RemotingException, MQBrokerException, InterruptedException { + // 添加虚拟运行环境相关的projectGroupPrefix + if (!UtilAll.isBlank(projectGroupPrefix)) { + requestHeader.setConsumerGroup(VirtualEnvUtil.buildWithProjectGroup( + requestHeader.getConsumerGroup(), projectGroupPrefix)); + requestHeader.setTopic(VirtualEnvUtil.buildWithProjectGroup(requestHeader.getTopic(), + projectGroupPrefix)); + } + + RemotingCommand request = + RemotingCommand.createRequestCommand(RequestCode.QUERY_CONSUMER_OFFSET, requestHeader); + + RemotingCommand response = this.remotingClient.invokeSync(addr, request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + QueryConsumerOffsetResponseHeader responseHeader = + (QueryConsumerOffsetResponseHeader) response + .decodeCommandCustomHeader(QueryConsumerOffsetResponseHeader.class); + + return responseHeader.getOffset(); + } + default: + break; + } + + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + + + /** + * 更新Consumer消费进度 + */ + public void updateConsumerOffset(// + final String addr,// + final UpdateConsumerOffsetRequestHeader requestHeader,// + final long timeoutMillis// + ) throws RemotingException, MQBrokerException, InterruptedException { + // 添加虚拟运行环境相关的projectGroupPrefix + if (!UtilAll.isBlank(projectGroupPrefix)) { + requestHeader.setConsumerGroup(VirtualEnvUtil.buildWithProjectGroup( + requestHeader.getConsumerGroup(), projectGroupPrefix)); + requestHeader.setTopic(VirtualEnvUtil.buildWithProjectGroup(requestHeader.getTopic(), + projectGroupPrefix)); + } + + RemotingCommand request = + RemotingCommand.createRequestCommand(RequestCode.UPDATE_CONSUMER_OFFSET, requestHeader); + + RemotingCommand response = this.remotingClient.invokeSync(addr, request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + return; + } + default: + break; + } + + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + + + /** + * 更新Consumer消费进度 + * + * @throws InterruptedException + * @throws RemotingSendRequestException + * @throws RemotingTimeoutException + * @throws RemotingTooMuchRequestException + * + * @throws RemotingConnectException + */ + public void updateConsumerOffsetOneway(// + final String addr,// + final UpdateConsumerOffsetRequestHeader requestHeader,// + final long timeoutMillis// + ) throws RemotingConnectException, RemotingTooMuchRequestException, RemotingTimeoutException, + RemotingSendRequestException, InterruptedException { + // 添加虚拟运行环境相关的projectGroupPrefix + if (!UtilAll.isBlank(projectGroupPrefix)) { + requestHeader.setConsumerGroup(VirtualEnvUtil.buildWithProjectGroup( + requestHeader.getConsumerGroup(), projectGroupPrefix)); + requestHeader.setTopic(VirtualEnvUtil.buildWithProjectGroup(requestHeader.getTopic(), + projectGroupPrefix)); + } + + RemotingCommand request = + RemotingCommand.createRequestCommand(RequestCode.UPDATE_CONSUMER_OFFSET, requestHeader); + + this.remotingClient.invokeOneway(addr, request, timeoutMillis); + } + + + /** + * 发送心跳 + */ + public void sendHearbeat(// + final String addr,// + final HeartbeatData heartbeatData,// + final long timeoutMillis// + ) throws RemotingException, MQBrokerException, InterruptedException { + // 添加虚拟运行环境相关的projectGroupPrefix + if (!UtilAll.isBlank(projectGroupPrefix)) { + Set consumerDatas = heartbeatData.getConsumerDataSet(); + for (ConsumerData consumerData : consumerDatas) { + consumerData.setGroupName(VirtualEnvUtil.buildWithProjectGroup(consumerData.getGroupName(), + projectGroupPrefix)); + Set subscriptionDatas = consumerData.getSubscriptionDataSet(); + for (SubscriptionData subscriptionData : subscriptionDatas) { + subscriptionData.setTopic(VirtualEnvUtil.buildWithProjectGroup( + subscriptionData.getTopic(), projectGroupPrefix)); + } + } + Set producerDatas = heartbeatData.getProducerDataSet(); + for (ProducerData producerData : producerDatas) { + producerData.setGroupName(VirtualEnvUtil.buildWithProjectGroup(producerData.getGroupName(), + projectGroupPrefix)); + } + } + + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.HEART_BEAT, null); + + request.setBody(heartbeatData.encode()); + RemotingCommand response = this.remotingClient.invokeSync(addr, request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + return; + } + default: + break; + } + + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + + + /** + * 发送心跳 + */ + public void unregisterClient(// + final String addr,// + final String clientID,// + final String producerGroup,// + final String consumerGroup,// + final long timeoutMillis// + ) throws RemotingException, MQBrokerException, InterruptedException { + // 添加虚拟运行环境相关的projectGroupPrefix + String producerGroupWithProjectGroup = producerGroup; + String consumerGroupWithProjectGroup = consumerGroup; + if (!UtilAll.isBlank(projectGroupPrefix)) { + producerGroupWithProjectGroup = + VirtualEnvUtil.buildWithProjectGroup(producerGroup, projectGroupPrefix); + consumerGroupWithProjectGroup = + VirtualEnvUtil.buildWithProjectGroup(consumerGroup, projectGroupPrefix); + } + + final UnregisterClientRequestHeader requestHeader = new UnregisterClientRequestHeader(); + requestHeader.setClientID(clientID); + requestHeader.setProducerGroup(producerGroupWithProjectGroup); + requestHeader.setConsumerGroup(consumerGroupWithProjectGroup); + RemotingCommand request = + RemotingCommand.createRequestCommand(RequestCode.UNREGISTER_CLIENT, requestHeader); + + RemotingCommand response = this.remotingClient.invokeSync(addr, request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + return; + } + default: + break; + } + + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + + + /** + * 提交或者回滚事务 + */ + public void endTransactionOneway(// + final String addr,// + final EndTransactionRequestHeader requestHeader,// + final String remark,// + final long timeoutMillis// + ) throws RemotingException, MQBrokerException, InterruptedException { + // 添加虚拟运行环境相关的projectGroupPrefix + if (!UtilAll.isBlank(projectGroupPrefix)) { + requestHeader.setProducerGroup(VirtualEnvUtil.buildWithProjectGroup( + requestHeader.getProducerGroup(), projectGroupPrefix)); + } + + RemotingCommand request = + RemotingCommand.createRequestCommand(RequestCode.END_TRANSACTION, requestHeader); + + request.setRemark(remark); + this.remotingClient.invokeOneway(addr, request, timeoutMillis); + } + + + /** + * 查询消息 + */ + public void queryMessage(// + final String addr,// + final QueryMessageRequestHeader requestHeader,// + final long timeoutMillis,// + final InvokeCallback invokeCallback// + ) throws RemotingException, MQBrokerException, InterruptedException { + // 添加虚拟运行环境相关的projectGroupPrefix + if (!UtilAll.isBlank(projectGroupPrefix)) { + requestHeader.setTopic(VirtualEnvUtil.buildWithProjectGroup(requestHeader.getTopic(), + projectGroupPrefix)); + } + + RemotingCommand request = + RemotingCommand.createRequestCommand(RequestCode.QUERY_MESSAGE, requestHeader); + + this.remotingClient.invokeAsync(addr, request, timeoutMillis, invokeCallback); + } + + + public boolean registerClient(final String addr, final HeartbeatData heartbeat, final long timeoutMillis) + throws RemotingException, InterruptedException { + // 添加虚拟运行环境相关的projectGroupPrefix + if (!UtilAll.isBlank(projectGroupPrefix)) { + Set consumerDatas = heartbeat.getConsumerDataSet(); + for (ConsumerData consumerData : consumerDatas) { + consumerData.setGroupName(VirtualEnvUtil.buildWithProjectGroup(consumerData.getGroupName(), + projectGroupPrefix)); + Set subscriptionDatas = consumerData.getSubscriptionDataSet(); + for (SubscriptionData subscriptionData : subscriptionDatas) { + subscriptionData.setTopic(VirtualEnvUtil.buildWithProjectGroup( + subscriptionData.getTopic(), projectGroupPrefix)); + } + } + Set producerDatas = heartbeat.getProducerDataSet(); + for (ProducerData producerData : producerDatas) { + producerData.setGroupName(VirtualEnvUtil.buildWithProjectGroup(producerData.getGroupName(), + projectGroupPrefix)); + } + } + + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.HEART_BEAT, null); + + request.setBody(heartbeat.encode()); + RemotingCommand response = this.remotingClient.invokeSync(addr, request, timeoutMillis); + return response.getCode() == ResponseCode.SUCCESS; + } + + + /** + * 失败的消息发回Broker + */ + public void consumerSendMessageBack(// + final String addr, // + final MessageExt msg,// + final String consumerGroup,// + final int delayLevel,// + final long timeoutMillis// + ) throws RemotingException, MQBrokerException, InterruptedException { + // 添加虚拟运行环境相关的projectGroupPrefix + String consumerGroupWithProjectGroup = consumerGroup; + if (!UtilAll.isBlank(projectGroupPrefix)) { + consumerGroupWithProjectGroup = + VirtualEnvUtil.buildWithProjectGroup(consumerGroup, projectGroupPrefix); + msg.setTopic(VirtualEnvUtil.buildWithProjectGroup(msg.getTopic(), projectGroupPrefix)); + } + + ConsumerSendMsgBackRequestHeader requestHeader = new ConsumerSendMsgBackRequestHeader(); + RemotingCommand request = + RemotingCommand.createRequestCommand(RequestCode.CONSUMER_SEND_MSG_BACK, requestHeader); + + requestHeader.setGroup(consumerGroupWithProjectGroup); + requestHeader.setOriginTopic(msg.getTopic()); + requestHeader.setOffset(msg.getCommitLogOffset()); + requestHeader.setDelayLevel(delayLevel); + requestHeader.setOriginMsgId(msg.getMsgId()); + + RemotingCommand response = this.remotingClient.invokeSync(addr, request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + return; + } + default: + break; + } + + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + + + public Set lockBatchMQ(// + final String addr,// + final LockBatchRequestBody requestBody,// + final long timeoutMillis) throws RemotingException, MQBrokerException, InterruptedException { + // 添加虚拟运行环境相关的projectGroupPrefix + if (!UtilAll.isBlank(projectGroupPrefix)) { + requestBody.setConsumerGroup((VirtualEnvUtil.buildWithProjectGroup( + requestBody.getConsumerGroup(), projectGroupPrefix))); + Set messageQueues = requestBody.getMqSet(); + for (MessageQueue messageQueue : messageQueues) { + messageQueue.setTopic(VirtualEnvUtil.buildWithProjectGroup(messageQueue.getTopic(), + projectGroupPrefix)); + } + } + + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.LOCK_BATCH_MQ, null); + + request.setBody(requestBody.encode()); + RemotingCommand response = this.remotingClient.invokeSync(addr, request, timeoutMillis); + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + LockBatchResponseBody responseBody = + LockBatchResponseBody.decode(response.getBody(), LockBatchResponseBody.class); + Set messageQueues = responseBody.getLockOKMQSet(); + // 清除虚拟运行环境相关的projectGroupPrefix + if (!UtilAll.isBlank(projectGroupPrefix)) { + for (MessageQueue messageQueue : messageQueues) { + messageQueue.setTopic(VirtualEnvUtil.clearProjectGroup(messageQueue.getTopic(), + projectGroupPrefix)); + } + } + return messageQueues; + } + default: + break; + } + + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + + + public void unlockBatchMQ(// + final String addr,// + final UnlockBatchRequestBody requestBody,// + final long timeoutMillis,// + final boolean oneway// + ) throws RemotingException, MQBrokerException, InterruptedException { + // 添加虚拟运行环境相关的projectGroupPrefix + if (!UtilAll.isBlank(projectGroupPrefix)) { + requestBody.setConsumerGroup(VirtualEnvUtil.buildWithProjectGroup(requestBody.getConsumerGroup(), + projectGroupPrefix)); + Set messageQueues = requestBody.getMqSet(); + for (MessageQueue messageQueue : messageQueues) { + messageQueue.setTopic(VirtualEnvUtil.buildWithProjectGroup(messageQueue.getTopic(), + projectGroupPrefix)); + } + } + + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UNLOCK_BATCH_MQ, null); + + request.setBody(requestBody.encode()); + + if (oneway) { + this.remotingClient.invokeOneway(addr, request, timeoutMillis); + } + else { + RemotingCommand response = this.remotingClient.invokeSync(addr, request, timeoutMillis); + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + return; + } + default: + break; + } + + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + } + + + public TopicStatsTable getTopicStatsInfo(final String addr, final String topic, final long timeoutMillis) + throws InterruptedException, RemotingTimeoutException, RemotingSendRequestException, + RemotingConnectException, MQBrokerException { + // 添加虚拟运行环境相关的projectGroupPrefix + String topicWithProjectGroup = topic; + if (!UtilAll.isBlank(projectGroupPrefix)) { + topicWithProjectGroup = VirtualEnvUtil.buildWithProjectGroup(topic, projectGroupPrefix); + } + + GetTopicStatsInfoRequestHeader requestHeader = new GetTopicStatsInfoRequestHeader(); + requestHeader.setTopic(topicWithProjectGroup); + + RemotingCommand request = + RemotingCommand.createRequestCommand(RequestCode.GET_TOPIC_STATS_INFO, requestHeader); + + RemotingCommand response = this.remotingClient.invokeSync(addr, request, timeoutMillis); + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + TopicStatsTable topicStatsTable = + TopicStatsTable.decode(response.getBody(), TopicStatsTable.class); + // 清除虚拟运行环境相关的projectGroupPrefix + if (!UtilAll.isBlank(projectGroupPrefix)) { + HashMap newTopicOffsetMap = + new HashMap(); + for (Map.Entry messageQueue : topicStatsTable.getOffsetTable() + .entrySet()) { + MessageQueue key = messageQueue.getKey(); + key.setTopic(VirtualEnvUtil.clearProjectGroup(key.getTopic(), projectGroupPrefix)); + newTopicOffsetMap.put(key, messageQueue.getValue()); + } + topicStatsTable.setOffsetTable(newTopicOffsetMap); + } + return topicStatsTable; + } + default: + break; + } + + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + + + public ConsumeStats getConsumeStats(final String addr, final String consumerGroup, + final long timeoutMillis) throws InterruptedException, RemotingTimeoutException, + RemotingSendRequestException, RemotingConnectException, MQBrokerException { + // 添加虚拟运行环境相关的projectGroupPrefix + String consumerGroupWithProjectGroup = consumerGroup; + if (!UtilAll.isBlank(projectGroupPrefix)) { + consumerGroupWithProjectGroup = + VirtualEnvUtil.buildWithProjectGroup(consumerGroup, projectGroupPrefix); + } + + GetConsumeStatsRequestHeader requestHeader = new GetConsumeStatsRequestHeader(); + requestHeader.setConsumerGroup(consumerGroupWithProjectGroup); + + RemotingCommand request = + RemotingCommand.createRequestCommand(RequestCode.GET_CONSUME_STATS, requestHeader); + + RemotingCommand response = this.remotingClient.invokeSync(addr, request, timeoutMillis); + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + ConsumeStats consumeStats = ConsumeStats.decode(response.getBody(), ConsumeStats.class); + // 清除虚拟运行环境相关的projectGroupPrefix + if (!UtilAll.isBlank(projectGroupPrefix)) { + HashMap newTopicOffsetMap = + new HashMap(); + for (Map.Entry messageQueue : consumeStats.getOffsetTable() + .entrySet()) { + MessageQueue key = messageQueue.getKey(); + key.setTopic(VirtualEnvUtil.clearProjectGroup(key.getTopic(), projectGroupPrefix)); + newTopicOffsetMap.put(key, messageQueue.getValue()); + } + consumeStats.setOffsetTable(newTopicOffsetMap); + } + + return consumeStats; + } + default: + break; + } + + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + + + /** + * 根据ProducerGroup获取Producer连接列表 + */ + public ProducerConnection getProducerConnectionList(final String addr, final String producerGroup, + final long timeoutMillis) throws RemotingConnectException, RemotingSendRequestException, + RemotingTimeoutException, InterruptedException, MQBrokerException { + // 添加虚拟运行环境相关的projectGroupPrefix + String producerGroupWithProjectGroup = producerGroup; + if (!UtilAll.isBlank(projectGroupPrefix)) { + producerGroupWithProjectGroup = + VirtualEnvUtil.buildWithProjectGroup(producerGroup, projectGroupPrefix); + } + + GetProducerConnectionListRequestHeader requestHeader = new GetProducerConnectionListRequestHeader(); + requestHeader.setProducerGroup(producerGroupWithProjectGroup); + + RemotingCommand request = + RemotingCommand.createRequestCommand(RequestCode.GET_PRODUCER_CONNECTION_LIST, requestHeader); + + RemotingCommand response = this.remotingClient.invokeSync(addr, request, timeoutMillis); + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + return ProducerConnection.decode(response.getBody(), ProducerConnection.class); + } + default: + break; + } + + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + + + /** + * 根据ConsumerGroup获取Consumer连接列表以及订阅关系 + */ + public ConsumerConnection getConsumerConnectionList(final String addr, final String consumerGroup, + final long timeoutMillis) throws RemotingConnectException, RemotingSendRequestException, + RemotingTimeoutException, InterruptedException, MQBrokerException { + // 添加虚拟运行环境相关的projectGroupPrefix + String consumerGroupWithProjectGroup = consumerGroup; + if (!UtilAll.isBlank(projectGroupPrefix)) { + consumerGroupWithProjectGroup = + VirtualEnvUtil.buildWithProjectGroup(consumerGroup, projectGroupPrefix); + } + + GetConsumerConnectionListRequestHeader requestHeader = new GetConsumerConnectionListRequestHeader(); + requestHeader.setConsumerGroup(consumerGroupWithProjectGroup); + + RemotingCommand request = + RemotingCommand.createRequestCommand(RequestCode.GET_CONSUMER_CONNECTION_LIST, requestHeader); + + RemotingCommand response = this.remotingClient.invokeSync(addr, request, timeoutMillis); + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + ConsumerConnection consumerConnection = + ConsumerConnection.decode(response.getBody(), ConsumerConnection.class); + if (!UtilAll.isBlank(projectGroupPrefix)) { + ConcurrentHashMap subscriptionDataConcurrentHashMap = + consumerConnection.getSubscriptionTable(); + for (Map.Entry subscriptionDataEntry : subscriptionDataConcurrentHashMap + .entrySet()) { + SubscriptionData subscriptionData = subscriptionDataEntry.getValue(); + subscriptionDataEntry.getValue().setTopic( + VirtualEnvUtil.clearProjectGroup(subscriptionData.getTopic(), projectGroupPrefix)); + } + } + return consumerConnection; + } + default: + break; + } + + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + + + public KVTable getBrokerRuntimeInfo(final String addr, final long timeoutMillis) + throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, + InterruptedException, MQBrokerException { + + RemotingCommand request = + RemotingCommand.createRequestCommand(RequestCode.GET_BROKER_RUNTIME_INFO, null); + + RemotingCommand response = this.remotingClient.invokeSync(addr, request, timeoutMillis); + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + return KVTable.decode(response.getBody(), KVTable.class); + } + default: + break; + } + + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + + + /** + * 更新Broker的配置文件 + * + * @param addr + * @param properties + * @param timeoutMillis + * @throws RemotingConnectException + * @throws RemotingSendRequestException + * @throws RemotingTimeoutException + * @throws InterruptedException + * @throws MQBrokerException + * @throws UnsupportedEncodingException + */ + public void updateBrokerConfig(final String addr, final Properties properties, final long timeoutMillis) + throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, + InterruptedException, MQBrokerException, UnsupportedEncodingException { + + RemotingCommand request = + RemotingCommand.createRequestCommand(RequestCode.UPDATE_BROKER_CONFIG, null); + + String str = MixAll.properties2String(properties); + if (str != null && str.length() > 0) { + request.setBody(str.getBytes(MixAll.DEFAULT_CHARSET)); + RemotingCommand response = this.remotingClient.invokeSync(addr, request, timeoutMillis); + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + return; + } + default: + break; + } + + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + } + + + /** + * Name Server: 从Name Server获取集群信息 + */ + public ClusterInfo getBrokerClusterInfo(final long timeoutMillis) throws InterruptedException, + RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException, + MQBrokerException { + RemotingCommand request = + RemotingCommand.createRequestCommand(RequestCode.GET_BROKER_CLUSTER_INFO, null); + + RemotingCommand response = this.remotingClient.invokeSync(null, request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + ClusterInfo responseBody = ClusterInfo.decode(response.getBody(), ClusterInfo.class); + return responseBody; + } + default: + break; + } + + throw new MQBrokerException(response.getCode(), response.getRemark()); + + } + + + /** + * Name Server: 从Name Server获取 Default Topic 路由信息 + */ + public TopicRouteData getDefaultTopicRouteInfoFromNameServer(final String topic, final long timeoutMillis) + throws RemotingException, MQClientException, InterruptedException { + GetRouteInfoRequestHeader requestHeader = new GetRouteInfoRequestHeader(); + requestHeader.setTopic(topic); + + RemotingCommand request = + RemotingCommand.createRequestCommand(RequestCode.GET_ROUTEINTO_BY_TOPIC, requestHeader); + + RemotingCommand response = this.remotingClient.invokeSync(null, request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.TOPIC_NOT_EXIST: { + // TODO LOG + break; + } + case ResponseCode.SUCCESS: { + byte[] body = response.getBody(); + if (body != null) { + return TopicRouteData.decode(body, TopicRouteData.class); + } + } + default: + break; + } + + throw new MQClientException(response.getCode(), response.getRemark()); + } + + + /** + * Name Server: 从Name Server获取Topic路由信息 + */ + public TopicRouteData getTopicRouteInfoFromNameServer(final String topic, final long timeoutMillis) + throws RemotingException, MQClientException, InterruptedException { + // 添加虚拟运行环境相关的projectGroupPrefix + String topicWithProjectGroup = topic; + if (!UtilAll.isBlank(projectGroupPrefix)) { + topicWithProjectGroup = VirtualEnvUtil.buildWithProjectGroup(topic, projectGroupPrefix); + } + + GetRouteInfoRequestHeader requestHeader = new GetRouteInfoRequestHeader(); + requestHeader.setTopic(topicWithProjectGroup); + + RemotingCommand request = + RemotingCommand.createRequestCommand(RequestCode.GET_ROUTEINTO_BY_TOPIC, requestHeader); + + RemotingCommand response = this.remotingClient.invokeSync(null, request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.TOPIC_NOT_EXIST: { + log.warn("get Topic [{}] RouteInfoFromNameServer is not exist value", topic); + break; + } + case ResponseCode.SUCCESS: { + byte[] body = response.getBody(); + if (body != null) { + return TopicRouteData.decode(body, TopicRouteData.class); + } + } + default: + break; + } + + throw new MQClientException(response.getCode(), response.getRemark()); + } + + + /** + * Name Server: 从Name Server获取所有Topic列表 + */ + public TopicList getTopicListFromNameServer(final long timeoutMillis) throws RemotingException, + MQClientException, InterruptedException { + RemotingCommand request = + RemotingCommand.createRequestCommand(RequestCode.GET_ALL_TOPIC_LIST_FROM_NAMESERVER, null); + + RemotingCommand response = this.remotingClient.invokeSync(null, request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + byte[] body = response.getBody(); + if (body != null) { + TopicList topicList = TopicList.decode(body, TopicList.class); + + if (!UtilAll.isBlank(projectGroupPrefix)) { + HashSet newTopicSet = new HashSet(); + for (String topic : topicList.getTopicList()) { + newTopicSet.add(VirtualEnvUtil.clearProjectGroup(topic, projectGroupPrefix)); + } + topicList.setTopicList(newTopicSet); + } + return topicList; + } + } + default: + break; + } + + throw new MQClientException(response.getCode(), response.getRemark()); + } + + + /** + * Name Server: Broker下线前，清除Broker对应的权限 + */ + public int wipeWritePermOfBroker(final String namesrvAddr, String brokerName, final long timeoutMillis) + throws RemotingCommandException, RemotingConnectException, RemotingSendRequestException, + RemotingTimeoutException, InterruptedException, MQClientException { + WipeWritePermOfBrokerRequestHeader requestHeader = new WipeWritePermOfBrokerRequestHeader(); + requestHeader.setBrokerName(brokerName); + + RemotingCommand request = + RemotingCommand.createRequestCommand(RequestCode.WIPE_WRITE_PERM_OF_BROKER, requestHeader); + + RemotingCommand response = this.remotingClient.invokeSync(namesrvAddr, request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + WipeWritePermOfBrokerResponseHeader responseHeader = + (WipeWritePermOfBrokerResponseHeader) response + .decodeCommandCustomHeader(WipeWritePermOfBrokerResponseHeader.class); + return responseHeader.getWipeTopicCount(); + } + default: + break; + } + + throw new MQClientException(response.getCode(), response.getRemark()); + } + + + public void deleteTopicInBroker(final String addr, final String topic, final long timeoutMillis) + throws RemotingException, MQBrokerException, InterruptedException, MQClientException { + // 添加虚拟运行环境相关的projectGroupPrefix + String topicWithProjectGroup = topic; + if (!UtilAll.isBlank(projectGroupPrefix)) { + topicWithProjectGroup = VirtualEnvUtil.buildWithProjectGroup(topic, projectGroupPrefix); + } + + DeleteTopicRequestHeader requestHeader = new DeleteTopicRequestHeader(); + requestHeader.setTopic(topicWithProjectGroup); + RemotingCommand request = + RemotingCommand.createRequestCommand(RequestCode.DELETE_TOPIC_IN_BROKER, requestHeader); + + RemotingCommand response = this.remotingClient.invokeSync(addr, request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + return; + } + default: + break; + } + + throw new MQClientException(response.getCode(), response.getRemark()); + } + + + public void deleteTopicInNameServer(final String addr, final String topic, final long timeoutMillis) + throws RemotingException, MQBrokerException, InterruptedException, MQClientException { + // 添加虚拟运行环境相关的projectGroupPrefix + String topicWithProjectGroup = topic; + if (!UtilAll.isBlank(projectGroupPrefix)) { + topicWithProjectGroup = VirtualEnvUtil.buildWithProjectGroup(topic, projectGroupPrefix); + } + + DeleteTopicRequestHeader requestHeader = new DeleteTopicRequestHeader(); + requestHeader.setTopic(topicWithProjectGroup); + RemotingCommand request = + RemotingCommand.createRequestCommand(RequestCode.DELETE_TOPIC_IN_NAMESRV, requestHeader); + + RemotingCommand response = this.remotingClient.invokeSync(addr, request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + return; + } + default: + break; + } + + throw new MQClientException(response.getCode(), response.getRemark()); + } + + + public void deleteSubscriptionGroup(final String addr, final String groupName, final long timeoutMillis) + throws RemotingException, MQBrokerException, InterruptedException, MQClientException { + // 添加虚拟运行环境相关的projectGroupPrefix + String groupWithProjectGroup = groupName; + if (!UtilAll.isBlank(projectGroupPrefix)) { + groupWithProjectGroup = VirtualEnvUtil.buildWithProjectGroup(groupName, projectGroupPrefix); + } + + DeleteSubscriptionGroupRequestHeader requestHeader = new DeleteSubscriptionGroupRequestHeader(); + requestHeader.setGroupName(groupWithProjectGroup); + RemotingCommand request = + RemotingCommand.createRequestCommand(RequestCode.DELETE_SUBSCRIPTIONGROUP, requestHeader); + + RemotingCommand response = this.remotingClient.invokeSync(addr, request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + return; + } + default: + break; + } + + throw new MQClientException(response.getCode(), response.getRemark()); + } + + + /** + * Name Server: 从Namesrv获取KV配置 + */ + public String getKVConfigValue(final String namespace, final String key, final long timeoutMillis) + throws RemotingException, MQClientException, InterruptedException { + GetKVConfigRequestHeader requestHeader = new GetKVConfigRequestHeader(); + requestHeader.setNamespace(namespace); + requestHeader.setKey(key); + + RemotingCommand request = + RemotingCommand.createRequestCommand(RequestCode.GET_KV_CONFIG, requestHeader); + + RemotingCommand response = this.remotingClient.invokeSync(null, request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + GetKVConfigResponseHeader responseHeader = + (GetKVConfigResponseHeader) response + .decodeCommandCustomHeader(GetKVConfigResponseHeader.class); + return responseHeader.getValue(); + } + default: + break; + } + + throw new MQClientException(response.getCode(), response.getRemark()); + } + + + /** + * Name Server: 添加KV配置 + */ + public void putKVConfigValue(final String namespace, final String key, final String value, + final long timeoutMillis) throws RemotingException, MQClientException, InterruptedException { + PutKVConfigRequestHeader requestHeader = new PutKVConfigRequestHeader(); + requestHeader.setNamespace(namespace); + requestHeader.setKey(key); + requestHeader.setValue(value); + + RemotingCommand request = + RemotingCommand.createRequestCommand(RequestCode.PUT_KV_CONFIG, requestHeader); + + List nameServerAddressList = this.remotingClient.getNameServerAddressList(); + if (nameServerAddressList != null) { + RemotingCommand errResponse = null; + for (String namesrvAddr : nameServerAddressList) { + RemotingCommand response = + this.remotingClient.invokeSync(namesrvAddr, request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + break; + } + default: + errResponse = response; + } + } + + if (errResponse != null) { + throw new MQClientException(errResponse.getCode(), errResponse.getRemark()); + } + } + } + + + /** + * Name Server: 添加KV配置 + */ + public void deleteKVConfigValue(final String namespace, final String key, final long timeoutMillis) + throws RemotingException, MQClientException, InterruptedException { + DeleteKVConfigRequestHeader requestHeader = new DeleteKVConfigRequestHeader(); + requestHeader.setNamespace(namespace); + requestHeader.setKey(key); + + RemotingCommand request = + RemotingCommand.createRequestCommand(RequestCode.DELETE_KV_CONFIG, requestHeader); + + List nameServerAddressList = this.remotingClient.getNameServerAddressList(); + if (nameServerAddressList != null) { + RemotingCommand errResponse = null; + for (String namesrvAddr : nameServerAddressList) { + RemotingCommand response = + this.remotingClient.invokeSync(namesrvAddr, request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + break; + } + default: + errResponse = response; + } + } + if (errResponse != null) { + throw new MQClientException(errResponse.getCode(), errResponse.getRemark()); + } + } + } + + + /** + * Name Server: 通过 server ip 获取 project 信息 + */ + public String getProjectGroupByIp(String ip, final long timeoutMillis) throws RemotingException, + MQClientException, InterruptedException { + return getKVConfigValue(NamesrvUtil.NAMESPACE_PROJECT_CONFIG, ip, timeoutMillis); + } + + + /** + * Name Server: 通过 value 获取所有的 key 信息 + */ + public String getKVConfigByValue(final String namespace, String value, final long timeoutMillis) + throws RemotingException, MQClientException, InterruptedException { + GetKVConfigRequestHeader requestHeader = new GetKVConfigRequestHeader(); + requestHeader.setNamespace(namespace); + requestHeader.setKey(value); + + RemotingCommand request = + RemotingCommand.createRequestCommand(RequestCode.GET_KV_CONFIG_BY_VALUE, requestHeader); + + RemotingCommand response = this.remotingClient.invokeSync(null, request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + GetKVConfigResponseHeader responseHeader = + (GetKVConfigResponseHeader) response + .decodeCommandCustomHeader(GetKVConfigResponseHeader.class); + return responseHeader.getValue(); + } + default: + break; + } + + throw new MQClientException(response.getCode(), response.getRemark()); + } + + + /** + * Name Server: 获取指定Namespace下的所有KV + */ + public KVTable getKVListByNamespace(final String namespace, final long timeoutMillis) + throws RemotingException, MQClientException, InterruptedException { + GetKVListByNamespaceRequestHeader requestHeader = new GetKVListByNamespaceRequestHeader(); + requestHeader.setNamespace(namespace); + + RemotingCommand request = + RemotingCommand.createRequestCommand(RequestCode.GET_KVLIST_BY_NAMESPACE, requestHeader); + + RemotingCommand response = this.remotingClient.invokeSync(null, request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + return KVTable.decode(response.getBody(), KVTable.class); + } + default: + break; + } + + throw new MQClientException(response.getCode(), response.getRemark()); + } + + + /** + * Name Server: 删除 value 对应的所有 key + */ + public void deleteKVConfigByValue(final String namespace, final String projectGroup, + final long timeoutMillis) throws RemotingException, MQClientException, InterruptedException { + DeleteKVConfigRequestHeader requestHeader = new DeleteKVConfigRequestHeader(); + requestHeader.setNamespace(namespace); + requestHeader.setKey(projectGroup); + + RemotingCommand request = + RemotingCommand.createRequestCommand(RequestCode.DELETE_KV_CONFIG_BY_VALUE, requestHeader); + + List nameServerAddressList = this.remotingClient.getNameServerAddressList(); + + if (nameServerAddressList != null) { + RemotingCommand errResponse = null; + for (String namesrvAddr : nameServerAddressList) { + RemotingCommand response = + this.remotingClient.invokeSync(namesrvAddr, request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + break; + } + default: + errResponse = response; + } + } + if (errResponse != null) { + throw new MQClientException(errResponse.getCode(), errResponse.getRemark()); + } + } + } + + + /** + * 通知 broker 重置 offset + */ + public Map invokeBrokerToResetOffset(final String addr, final String topic, + final String group, final long timestamp, final boolean isForce, final long timeoutMillis) + throws RemotingException, MQClientException, InterruptedException { + ResetOffsetRequestHeader requestHeader = new ResetOffsetRequestHeader(); + requestHeader.setTopic(topic); + requestHeader.setGroup(group); + requestHeader.setTimestamp(timestamp); + requestHeader.setForce(isForce); + + RemotingCommand request = + RemotingCommand + .createRequestCommand(RequestCode.INVOKE_BROKER_TO_RESET_OFFSET, requestHeader); + + RemotingCommand response = this.remotingClient.invokeSync(addr, request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + if (response.getBody() != null) { + ResetOffsetBody body = ResetOffsetBody.decode(response.getBody(), ResetOffsetBody.class); + return body.getOffsetTable(); + } + } + default: + break; + } + + throw new MQClientException(response.getCode(), response.getRemark()); + } + + + /** + * 通知 broker 客户端订阅消息处理 + */ + public Map> invokeBrokerToGetConsumerStatus(final String addr, + final String topic, final String group, final String clientAddr, final long timeoutMillis) + throws RemotingException, MQClientException, InterruptedException { + GetConsumerStatusRequestHeader requestHeader = new GetConsumerStatusRequestHeader(); + requestHeader.setTopic(topic); + requestHeader.setGroup(group); + requestHeader.setClientAddr(clientAddr); + + RemotingCommand request = + RemotingCommand.createRequestCommand(RequestCode.INVOKE_BROKER_TO_GET_CONSUMER_STATUS, + requestHeader); + + RemotingCommand response = this.remotingClient.invokeSync(addr, request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + if (response.getBody() != null) { + GetConsumerStatusBody body = + GetConsumerStatusBody.decode(response.getBody(), GetConsumerStatusBody.class); + return body.getConsumerTable(); + } + } + default: + break; + } + + throw new MQClientException(response.getCode(), response.getRemark()); + } + + + /** + * 根据ConsumerGroup获取Consumer连接列表以及订阅关系 + */ + public GroupList queryTopicConsumeByWho(final String addr, final String topic, final long timeoutMillis) + throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, + InterruptedException, MQBrokerException { + QueryTopicConsumeByWhoRequestHeader requestHeader = new QueryTopicConsumeByWhoRequestHeader(); + requestHeader.setTopic(topic); + + RemotingCommand request = + RemotingCommand.createRequestCommand(RequestCode.QUERY_TOPIC_CONSUME_BY_WHO, requestHeader); + + RemotingCommand response = this.remotingClient.invokeSync(addr, request, timeoutMillis); + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + GroupList groupList = GroupList.decode(response.getBody(), GroupList.class); + return groupList; + } + default: + break; + } + + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + + + /** + * 根据 topic 和 group 获取消息的时间跨度 + */ + public Set queryConsumeTimeSpan(final String addr, final String topic, final String group, + final long timeoutMillis) throws RemotingConnectException, RemotingSendRequestException, + RemotingTimeoutException, InterruptedException, MQBrokerException { + QueryConsumeTimeSpanRequestHeader requestHeader = new QueryConsumeTimeSpanRequestHeader(); + requestHeader.setTopic(topic); + requestHeader.setGroup(group); + + RemotingCommand request = + RemotingCommand.createRequestCommand(RequestCode.QUERY_CONSUME_TIME_SPAN, requestHeader); + + RemotingCommand response = this.remotingClient.invokeSync(addr, request, timeoutMillis); + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + QueryConsumeTimeSpanBody consumeTimeSpanBody = + GroupList.decode(response.getBody(), QueryConsumeTimeSpanBody.class); + return consumeTimeSpanBody.getConsumeTimeSpanSet(); + } + default: + break; + } + + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + + + /** + * Name Server: 获取指定集群下的所有 topic + */ + public TopicList getTopicsByCluster(final String cluster, final long timeoutMillis) + throws RemotingException, MQClientException, InterruptedException { + GetTopicsByClusterRequestHeader requestHeader = new GetTopicsByClusterRequestHeader(); + requestHeader.setCluster(cluster); + RemotingCommand request = + RemotingCommand.createRequestCommand(RequestCode.GET_TOPICS_BY_CLUSTER, requestHeader); + + RemotingCommand response = this.remotingClient.invokeSync(null, request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + byte[] body = response.getBody(); + if (body != null) { + TopicList topicList = TopicList.decode(body, TopicList.class); + + if (!UtilAll.isBlank(projectGroupPrefix)) { + HashSet newTopicSet = new HashSet(); + for (String topic : topicList.getTopicList()) { + newTopicSet.add(VirtualEnvUtil.clearProjectGroup(topic, projectGroupPrefix)); + } + topicList.setTopicList(newTopicSet); + } + return topicList; + } + } + default: + break; + } + + throw new MQClientException(response.getCode(), response.getRemark()); + } + + + /** + * Filter Server: 向Filter Server注册过滤类 + * + * @throws InterruptedException + * @throws RemotingTimeoutException + * @throws RemotingSendRequestException + * @throws RemotingConnectException + * @throws MQBrokerException + */ + public void registerMessageFilterClass(final String addr,// + final String consumerGroup,// + final String topic,// + final String className,// + final int classCRC,// + final byte[] classBody,// + final long timeoutMillis) throws RemotingConnectException, RemotingSendRequestException, + RemotingTimeoutException, InterruptedException, MQBrokerException { + RegisterMessageFilterClassRequestHeader requestHeader = new RegisterMessageFilterClassRequestHeader(); + requestHeader.setConsumerGroup(consumerGroup); + requestHeader.setClassName(className); + requestHeader.setTopic(topic); + requestHeader.setClassCRC(classCRC); + + RemotingCommand request = + RemotingCommand + .createRequestCommand(RequestCode.REGISTER_MESSAGE_FILTER_CLASS, requestHeader); + request.setBody(classBody); + RemotingCommand response = this.remotingClient.invokeSync(addr, request, timeoutMillis); + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + return; + } + default: + break; + } + + throw new MQBrokerException(response.getCode(), response.getRemark()); + } + + + /** + * 获取所有系统内置 Topic 列表 + */ + public TopicList getSystemTopicList(final long timeoutMillis) throws RemotingException, + MQClientException, InterruptedException { + RemotingCommand request = + RemotingCommand.createRequestCommand(RequestCode.GET_SYSTEM_TOPIC_LIST_FROM_NS, null); + + RemotingCommand response = this.remotingClient.invokeSync(null, request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + byte[] body = response.getBody(); + if (body != null) { + TopicList topicList = TopicList.decode(response.getBody(), TopicList.class); + if (!UtilAll.isBlank(projectGroupPrefix)) { + HashSet newTopicSet = new HashSet(); + for (String topic : topicList.getTopicList()) { + newTopicSet.add(VirtualEnvUtil.clearProjectGroup(topic, projectGroupPrefix)); + } + topicList.setTopicList(newTopicSet); + } + + if (topicList.getTopicList() != null && !topicList.getTopicList().isEmpty() + && !UtilAll.isBlank(topicList.getBrokerAddr())) { + TopicList tmp = getSystemTopicListFromBroker(topicList.getBrokerAddr(), timeoutMillis); + if (tmp.getTopicList() != null && !tmp.getTopicList().isEmpty()) { + topicList.getTopicList().addAll(tmp.getTopicList()); + } + } + return topicList; + } + } + default: + break; + } + + throw new MQClientException(response.getCode(), response.getRemark()); + } + + + /** + * Name Server: 从Name Server获取所有系统内置 Topic 列表 + */ + public TopicList getSystemTopicListFromBroker(final String addr, final long timeoutMillis) + throws RemotingException, MQClientException, InterruptedException { + RemotingCommand request = + RemotingCommand.createRequestCommand(RequestCode.GET_SYSTEM_TOPIC_LIST_FROM_BROKER, null); + + RemotingCommand response = this.remotingClient.invokeSync(addr, request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + byte[] body = response.getBody(); + if (body != null) { + TopicList topicList = TopicList.decode(body, TopicList.class); + if (!UtilAll.isBlank(projectGroupPrefix)) { + HashSet newTopicSet = new HashSet(); + for (String topic : topicList.getTopicList()) { + newTopicSet.add(VirtualEnvUtil.clearProjectGroup(topic, projectGroupPrefix)); + } + topicList.setTopicList(newTopicSet); + } + return topicList; + } + } + default: + break; + } + + throw new MQClientException(response.getCode(), response.getRemark()); + } + + + public boolean cleanExpiredConsumeQueue(final String addr, long timeoutMillis) throws MQClientException, + RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, + InterruptedException { + RemotingCommand request = + RemotingCommand.createRequestCommand(RequestCode.CLEAN_EXPIRED_CONSUMEQUEUE, null); + RemotingCommand response = this.remotingClient.invokeSync(addr, request, timeoutMillis); + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + return true; + } + default: + break; + } + + throw new MQClientException(response.getCode(), response.getRemark()); + } + + + /** + * 通过调用Broker，从Consumer内存获取相应数据结构 + */ + public ConsumerRunningInfo getConsumerRunningInfo(final String addr, String consumerGroup, + String clientId, boolean jstack, final long timeoutMillis) throws RemotingException, + MQClientException, InterruptedException { + GetConsumerRunningInfoRequestHeader requestHeader = new GetConsumerRunningInfoRequestHeader(); + requestHeader.setConsumerGroup(consumerGroup); + requestHeader.setClientId(clientId); + requestHeader.setJstackEnable(jstack); + + RemotingCommand request = + RemotingCommand.createRequestCommand(RequestCode.GET_CONSUMER_RUNNING_INFO, requestHeader); + + RemotingCommand response = this.remotingClient.invokeSync(addr, request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + byte[] body = response.getBody(); + if (body != null) { + ConsumerRunningInfo info = ConsumerRunningInfo.decode(body, ConsumerRunningInfo.class); + return info; + } + } + default: + break; + } + + throw new MQClientException(response.getCode(), response.getRemark()); + } + + + /** + * 通过调用Broker，向指定Consumer发送某条消息，并返回消费结果 + */ + public ConsumeMessageDirectlyResult consumeMessageDirectly(final String addr, // + String consumerGroup, // + String clientId, // + String msgId, // + final long timeoutMillis) throws RemotingException, MQClientException, InterruptedException { + ConsumeMessageDirectlyResultRequestHeader requestHeader = + new ConsumeMessageDirectlyResultRequestHeader(); + requestHeader.setConsumerGroup(consumerGroup); + requestHeader.setClientId(clientId); + requestHeader.setMsgId(msgId); + + RemotingCommand request = + RemotingCommand.createRequestCommand(RequestCode.CONSUME_MESSAGE_DIRECTLY, requestHeader); + + RemotingCommand response = this.remotingClient.invokeSync(addr, request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + byte[] body = response.getBody(); + if (body != null) { + ConsumeMessageDirectlyResult info = + ConsumeMessageDirectlyResult.decode(body, ConsumeMessageDirectlyResult.class); + return info; + } + } + default: + break; + } + + throw new MQClientException(response.getCode(), response.getRemark()); + } + + + public String getProjectGroupPrefix() { + return projectGroupPrefix; + } + + + public Map queryCorrectionOffset(final String addr, final String topic, + final String group, Set filterGroup, long timeoutMillis) throws MQClientException, + RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, + InterruptedException { + QueryCorrectionOffsetHeader requestHeader = new QueryCorrectionOffsetHeader(); + requestHeader.setCompareGroup(group); + requestHeader.setTopic(topic); + if (filterGroup != null) { + StringBuilder sb = new StringBuilder(); + String splitor = ""; + for (String s : filterGroup) { + sb.append(splitor).append(s); + splitor = ","; + } + requestHeader.setFilterGroups(sb.toString()); + } + RemotingCommand request = + RemotingCommand.createRequestCommand(RequestCode.QUERY_CORRECTION_OFFSET, requestHeader); + + RemotingCommand response = this.remotingClient.invokeSync(addr, request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + if (response.getBody() != null) { + QueryCorrectionOffsetBody body = + QueryCorrectionOffsetBody.decode(response.getBody(), QueryCorrectionOffsetBody.class); + return body.getCorrectionOffsets(); + } + } + default: + break; + } + + throw new MQClientException(response.getCode(), response.getRemark()); + } + + + /** + * 获取单元化逻辑 Topic 列表 + */ + public TopicList getUnitTopicList(final boolean containRetry, final long timeoutMillis) + throws RemotingException, MQClientException, InterruptedException { + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_UNIT_TOPIC_LIST, null); + + RemotingCommand response = this.remotingClient.invokeSync(null, request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + byte[] body = response.getBody(); + if (body != null) { + TopicList topicList = TopicList.decode(response.getBody(), TopicList.class); + if (!UtilAll.isBlank(projectGroupPrefix)) { + HashSet newTopicSet = new HashSet(); + for (String topic : topicList.getTopicList()) { + newTopicSet.add(VirtualEnvUtil.clearProjectGroup(topic, projectGroupPrefix)); + } + topicList.setTopicList(newTopicSet); + } + if (!containRetry) { + Iterator it = topicList.getTopicList().iterator(); + while (it.hasNext()) { + String topic = it.next(); + if (topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) + it.remove(); + } + } + + return topicList; + } + } + default: + break; + } + + throw new MQClientException(response.getCode(), response.getRemark()); + } + + + /** + * 获取含有单元化订阅组的 Topic 列表 + */ + public TopicList getHasUnitSubTopicList(final boolean containRetry, final long timeoutMillis) + throws RemotingException, MQClientException, InterruptedException { + RemotingCommand request = + RemotingCommand.createRequestCommand(RequestCode.GET_HAS_UNIT_SUB_TOPIC_LIST, null); + + RemotingCommand response = this.remotingClient.invokeSync(null, request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + byte[] body = response.getBody(); + if (body != null) { + TopicList topicList = TopicList.decode(response.getBody(), TopicList.class); + if (!UtilAll.isBlank(projectGroupPrefix)) { + HashSet newTopicSet = new HashSet(); + for (String topic : topicList.getTopicList()) { + newTopicSet.add(VirtualEnvUtil.clearProjectGroup(topic, projectGroupPrefix)); + } + topicList.setTopicList(newTopicSet); + } + if (!containRetry) { + Iterator it = topicList.getTopicList().iterator(); + while (it.hasNext()) { + String topic = it.next(); + if (topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) + it.remove(); + } + } + return topicList; + } + } + default: + break; + } + + throw new MQClientException(response.getCode(), response.getRemark()); + } + + + /** + * 获取含有单元化订阅组的非单元化 Topic 列表 + */ + public TopicList getHasUnitSubUnUnitTopicList(final boolean containRetry, final long timeoutMillis) + throws RemotingException, MQClientException, InterruptedException { + RemotingCommand request = + RemotingCommand.createRequestCommand(RequestCode.GET_HAS_UNIT_SUB_UNUNIT_TOPIC_LIST, null); + + RemotingCommand response = this.remotingClient.invokeSync(null, request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + byte[] body = response.getBody(); + if (body != null) { + TopicList topicList = TopicList.decode(response.getBody(), TopicList.class); + if (!UtilAll.isBlank(projectGroupPrefix)) { + HashSet newTopicSet = new HashSet(); + for (String topic : topicList.getTopicList()) { + newTopicSet.add(VirtualEnvUtil.clearProjectGroup(topic, projectGroupPrefix)); + } + topicList.setTopicList(newTopicSet); + } + if (!containRetry) { + Iterator it = topicList.getTopicList().iterator(); + while (it.hasNext()) { + String topic = it.next(); + if (topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) + it.remove(); + } + } + return topicList; + } + } + default: + break; + } + + throw new MQClientException(response.getCode(), response.getRemark()); + } + + + /** + * 克隆某一个组的消费进度到新的组 + */ + public void cloneGroupOffset(final String addr, final String srcGroup, final String destGroup, + final String topic, final boolean isOffline, final long timeoutMillis) throws RemotingException, + MQClientException, InterruptedException { + CloneGroupOffsetRequestHeader requestHeader = new CloneGroupOffsetRequestHeader(); + requestHeader.setSrcGroup(srcGroup); + requestHeader.setDestGroup(destGroup); + requestHeader.setTopic(topic); + requestHeader.setOffline(isOffline); + RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CLONE_GROUP_OFFSET, null); + + RemotingCommand response = this.remotingClient.invokeSync(addr, request, timeoutMillis); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + return; + } + default: + break; + } + + throw new MQClientException(response.getCode(), response.getRemark()); + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/MQClientManager.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/MQClientManager.java new file mode 100644 index 000000000..01e82507f --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/MQClientManager.java @@ -0,0 +1,77 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.impl; + +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; + +import com.alibaba.rocketmq.client.ClientConfig; +import com.alibaba.rocketmq.client.impl.factory.MQClientInstance; +import com.alibaba.rocketmq.remoting.RPCHook; + + +/** + * Client单例管理 + * + * @author shijia.wxr + * @since 2013-7-24 + */ +public class MQClientManager { + private static MQClientManager instance = new MQClientManager(); + private AtomicInteger factoryIndexGenerator = new AtomicInteger(); + private ConcurrentHashMap factoryTable = + new ConcurrentHashMap(); + + + private MQClientManager() { + + } + + + public static MQClientManager getInstance() { + return instance; + } + + + public MQClientInstance getAndCreateMQClientInstance(final ClientConfig clientConfig, RPCHook rpcHook) { + String clientId = clientConfig.buildMQClientId(); + MQClientInstance instance = this.factoryTable.get(clientId); + if (null == instance) { + instance = + new MQClientInstance(clientConfig.cloneClientConfig(), + this.factoryIndexGenerator.getAndIncrement(), clientId, rpcHook); + MQClientInstance prev = this.factoryTable.putIfAbsent(clientId, instance); + if (prev != null) { + instance = prev; + } + else { + // TODO log + } + } + + return instance; + } + + + public MQClientInstance getAndCreateMQClientInstance(final ClientConfig clientConfig) { + return getAndCreateMQClientInstance(clientConfig, null); + } + + + public void removeClientFactory(final String clientId) { + this.factoryTable.remove(clientId); + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/ConsumeMessageConcurrentlyService.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/ConsumeMessageConcurrentlyService.java new file mode 100644 index 000000000..0c61264d3 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/ConsumeMessageConcurrentlyService.java @@ -0,0 +1,460 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.impl.consumer; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +import org.slf4j.Logger; + +import com.alibaba.rocketmq.client.consumer.DefaultMQPushConsumer; +import com.alibaba.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext; +import com.alibaba.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus; +import com.alibaba.rocketmq.client.consumer.listener.MessageListenerConcurrently; +import com.alibaba.rocketmq.client.hook.ConsumeMessageContext; +import com.alibaba.rocketmq.client.log.ClientLogger; +import com.alibaba.rocketmq.client.stat.ConsumerStatsManager; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.ThreadFactoryImpl; +import com.alibaba.rocketmq.common.message.MessageConst; +import com.alibaba.rocketmq.common.message.MessageExt; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.common.protocol.body.CMResult; +import com.alibaba.rocketmq.common.protocol.body.ConsumeMessageDirectlyResult; +import com.alibaba.rocketmq.remoting.common.RemotingHelper; + + +/** + * 并发消费消息服务 + * + * @author shijia.wxr + * @since 2013-7-24 + */ +public class ConsumeMessageConcurrentlyService implements ConsumeMessageService { + private static final Logger log = ClientLogger.getLog(); + private final DefaultMQPushConsumerImpl defaultMQPushConsumerImpl; + private final DefaultMQPushConsumer defaultMQPushConsumer; + private final MessageListenerConcurrently messageListener; + private final BlockingQueue consumeRequestQueue; + private final ThreadPoolExecutor consumeExecutor; + private final String consumerGroup; + + // 定时线程 + private final ScheduledExecutorService scheduledExecutorService; + + + public ConsumeMessageConcurrentlyService(DefaultMQPushConsumerImpl defaultMQPushConsumerImpl, + MessageListenerConcurrently messageListener) { + this.defaultMQPushConsumerImpl = defaultMQPushConsumerImpl; + this.messageListener = messageListener; + + this.defaultMQPushConsumer = this.defaultMQPushConsumerImpl.getDefaultMQPushConsumer(); + this.consumerGroup = this.defaultMQPushConsumer.getConsumerGroup(); + this.consumeRequestQueue = new LinkedBlockingQueue(); + + this.consumeExecutor = new ThreadPoolExecutor(// + this.defaultMQPushConsumer.getConsumeThreadMin(),// + this.defaultMQPushConsumer.getConsumeThreadMax(),// + 1000 * 60,// + TimeUnit.MILLISECONDS,// + this.consumeRequestQueue,// + new ThreadFactoryImpl("ConsumeMessageThread_")); + + this.scheduledExecutorService = + Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl( + "ConsumeMessageScheduledThread_")); + } + + + public void start() { + } + + + public void shutdown() { + this.scheduledExecutorService.shutdown(); + this.consumeExecutor.shutdown(); + } + + + public ConsumerStatsManager getConsumerStatsManager() { + return this.defaultMQPushConsumerImpl.getConsumerStatsManager(); + } + + class ConsumeRequest implements Runnable { + private final List msgs; + private final ProcessQueue processQueue; + private final MessageQueue messageQueue; + + + public ConsumeRequest(List msgs, ProcessQueue processQueue, MessageQueue messageQueue) { + this.msgs = msgs; + this.processQueue = processQueue; + this.messageQueue = messageQueue; + } + + + @Override + public void run() { + if (this.processQueue.isDroped()) { + log.info("the message queue not be able to consume, because it's dropped {}", + this.messageQueue); + return; + } + + MessageListenerConcurrently listener = ConsumeMessageConcurrentlyService.this.messageListener; + ConsumeConcurrentlyContext context = new ConsumeConcurrentlyContext(messageQueue); + ConsumeConcurrentlyStatus status = null; + + // 执行Hook + ConsumeMessageContext consumeMessageContext = null; + if (ConsumeMessageConcurrentlyService.this.defaultMQPushConsumerImpl.hasHook()) { + consumeMessageContext = new ConsumeMessageContext(); + consumeMessageContext + .setConsumerGroup(ConsumeMessageConcurrentlyService.this.defaultMQPushConsumer + .getConsumerGroup()); + consumeMessageContext.setMq(messageQueue); + consumeMessageContext.setMsgList(msgs); + consumeMessageContext.setSuccess(false); + ConsumeMessageConcurrentlyService.this.defaultMQPushConsumerImpl + .executeHookBefore(consumeMessageContext); + } + + long beginTimestamp = System.currentTimeMillis(); + + try { + ConsumeMessageConcurrentlyService.this.resetRetryTopic(msgs); + status = listener.consumeMessage(Collections.unmodifiableList(msgs), context); + } + catch (Throwable e) { + log.warn("consumeMessage exception: {} Group: {} Msgs: {} MQ: {}",// + RemotingHelper.exceptionSimpleDesc(e),// + ConsumeMessageConcurrentlyService.this.consumerGroup,// + msgs,// + messageQueue); + } + + long consumeRT = System.currentTimeMillis() - beginTimestamp; + + if (null == status) { + log.warn("consumeMessage return null, Group: {} Msgs: {} MQ: {}",// + ConsumeMessageConcurrentlyService.this.consumerGroup,// + msgs,// + messageQueue); + status = ConsumeConcurrentlyStatus.RECONSUME_LATER; + } + + // 执行Hook + if (ConsumeMessageConcurrentlyService.this.defaultMQPushConsumerImpl.hasHook()) { + consumeMessageContext.setStatus(status.toString()); + consumeMessageContext.setSuccess(ConsumeConcurrentlyStatus.CONSUME_SUCCESS == status); + ConsumeMessageConcurrentlyService.this.defaultMQPushConsumerImpl + .executeHookAfter(consumeMessageContext); + } + + // 记录统计信息 + ConsumeMessageConcurrentlyService.this.getConsumerStatsManager().incConsumeRT( + ConsumeMessageConcurrentlyService.this.consumerGroup, messageQueue.getTopic(), consumeRT); + + // 如果ProcessQueue是dropped状态，不需要直接更新 offset + if (!processQueue.isDroped()) { + ConsumeMessageConcurrentlyService.this.processConsumeResult(status, context, this); + } + else { + log.warn( + "processQueue is dropped without process consume result. messageQueue={}, msgTreeMap={}, msgs={}", + new Object[] { messageQueue, processQueue.getMsgTreeMap(), msgs }); + } + } + + + public List getMsgs() { + return msgs; + } + + + public ProcessQueue getProcessQueue() { + return processQueue; + } + + + public MessageQueue getMessageQueue() { + return messageQueue; + } + } + + + public boolean sendMessageBack(final MessageExt msg, final ConsumeConcurrentlyContext context) { + // 如果用户没有设置，服务器会根据重试次数自动叠加延时时间 + int delayLevel = context.getDelayLevelWhenNextConsume(); + + try { + this.defaultMQPushConsumerImpl.sendMessageBack(msg, delayLevel, context.getMessageQueue() + .getBrokerName()); + return true; + } + catch (Exception e) { + log.error("sendMessageBack exception, group: " + this.consumerGroup + " msg: " + msg.toString(), + e); + } + + return false; + } + + + public void processConsumeResult(// + final ConsumeConcurrentlyStatus status, // + final ConsumeConcurrentlyContext context, // + final ConsumeRequest consumeRequest// + ) { + int ackIndex = context.getAckIndex(); + + if (consumeRequest.getMsgs().isEmpty()) + return; + + switch (status) { + case CONSUME_SUCCESS: + if (ackIndex >= consumeRequest.getMsgs().size()) { + ackIndex = consumeRequest.getMsgs().size() - 1; + } + int ok = ackIndex + 1; + int failed = consumeRequest.getMsgs().size() - ok; + // 统计信息 + this.getConsumerStatsManager().incConsumeOKTPS(consumerGroup, + consumeRequest.getMessageQueue().getTopic(), ok); + this.getConsumerStatsManager().incConsumeFailedTPS(consumerGroup, + consumeRequest.getMessageQueue().getTopic(), failed); + break; + case RECONSUME_LATER: + ackIndex = -1; + // 统计信息 + this.getConsumerStatsManager().incConsumeFailedTPS(consumerGroup, + consumeRequest.getMessageQueue().getTopic(), consumeRequest.getMsgs().size()); + break; + default: + break; + } + + switch (this.defaultMQPushConsumer.getMessageModel()) { + case BROADCASTING: + // 如果是广播模式，直接丢弃失败消息，需要在文档中告知用户 + // 这样做的原因：广播模式对于失败重试代价过高，对整个集群性能会有较大影响，失败重试功能交由应用处理 + for (int i = ackIndex + 1; i < consumeRequest.getMsgs().size(); i++) { + MessageExt msg = consumeRequest.getMsgs().get(i); + log.warn("BROADCASTING, the message consume failed, drop it, {}", msg.toString()); + } + break; + case CLUSTERING: + // 处理消费失败的消息，直接发回到Broker + List msgBackFailed = new ArrayList(consumeRequest.getMsgs().size()); + for (int i = ackIndex + 1; i < consumeRequest.getMsgs().size(); i++) { + MessageExt msg = consumeRequest.getMsgs().get(i); + boolean result = this.sendMessageBack(msg, context); + if (!result) { + msg.setReconsumeTimes(msg.getReconsumeTimes() + 1); + msgBackFailed.add(msg); + } + } + + if (!msgBackFailed.isEmpty()) { + // 发回失败的消息仍然要保留 + consumeRequest.getMsgs().removeAll(msgBackFailed); + + // 此过程处理失败的消息，需要在Client中做定时消费，直到成功 + this.submitConsumeRequestLater(msgBackFailed, consumeRequest.getProcessQueue(), + consumeRequest.getMessageQueue()); + } + break; + default: + break; + } + + long offset = consumeRequest.getProcessQueue().removeMessage(consumeRequest.getMsgs()); + if (offset >= 0) { + this.defaultMQPushConsumerImpl.getOffsetStore().updateOffset(consumeRequest.getMessageQueue(), + offset, true); + } + } + + + /** + * 在Consumer本地定时线程中定时重试 + */ + private void submitConsumeRequestLater(// + final List msgs, // + final ProcessQueue processQueue, // + final MessageQueue messageQueue// + ) { + + this.scheduledExecutorService.schedule(new Runnable() { + + @Override + public void run() { + ConsumeMessageConcurrentlyService.this.submitConsumeRequest(msgs, processQueue, messageQueue, + true); + } + }, 5000, TimeUnit.MILLISECONDS); + } + + + @Override + public void submitConsumeRequest(// + final List msgs, // + final ProcessQueue processQueue, // + final MessageQueue messageQueue, // + final boolean dispathToConsume) { + final int consumeBatchSize = this.defaultMQPushConsumer.getConsumeMessageBatchMaxSize(); + if (msgs.size() <= consumeBatchSize) { + ConsumeRequest consumeRequest = new ConsumeRequest(msgs, processQueue, messageQueue); + this.consumeExecutor.submit(consumeRequest); + } + else { + for (int total = 0; total < msgs.size();) { + List msgThis = new ArrayList(consumeBatchSize); + for (int i = 0; i < consumeBatchSize; i++, total++) { + if (total < msgs.size()) { + msgThis.add(msgs.get(total)); + } + else { + break; + } + } + + ConsumeRequest consumeRequest = new ConsumeRequest(msgThis, processQueue, messageQueue); + this.consumeExecutor.submit(consumeRequest); + } + } + } + + + @Override + public void updateCorePoolSize(int corePoolSize) { + if (corePoolSize > 0 // + && corePoolSize <= Short.MAX_VALUE // + && corePoolSize < this.defaultMQPushConsumer.getConsumeThreadMax()) { + this.consumeExecutor.setCorePoolSize(corePoolSize); + } + } + + + @Override + public void incCorePoolSize() { + long corePoolSize = this.consumeExecutor.getCorePoolSize(); + if (corePoolSize < this.defaultMQPushConsumer.getConsumeThreadMax()) { + this.consumeExecutor.setCorePoolSize(this.consumeExecutor.getCorePoolSize() + 1); + } + + log.info("incCorePoolSize Concurrently from {} to {}, ConsumerGroup: {}", // + corePoolSize,// + this.consumeExecutor.getCorePoolSize(),// + this.consumerGroup); + } + + + @Override + public void decCorePoolSize() { + long corePoolSize = this.consumeExecutor.getCorePoolSize(); + if (corePoolSize > this.defaultMQPushConsumer.getConsumeThreadMin()) { + this.consumeExecutor.setCorePoolSize(this.consumeExecutor.getCorePoolSize() - 1); + } + + log.info("decCorePoolSize Concurrently from {} to {}, ConsumerGroup: {}", // + corePoolSize,// + this.consumeExecutor.getCorePoolSize(),// + this.consumerGroup); + } + + + @Override + public int getCorePoolSize() { + return this.consumeExecutor.getCorePoolSize(); + } + + + @Override + public ConsumeMessageDirectlyResult consumeMessageDirectly(MessageExt msg, String brokerName) { + ConsumeMessageDirectlyResult result = new ConsumeMessageDirectlyResult(); + result.setOrder(false); + result.setAutoCommit(true); + + List msgs = new ArrayList(); + msgs.add(msg); + MessageQueue mq = new MessageQueue(); + mq.setBrokerName(brokerName); + mq.setTopic(msg.getTopic()); + mq.setQueueId(msg.getQueueId()); + + ConsumeConcurrentlyContext context = new ConsumeConcurrentlyContext(mq); + + this.resetRetryTopic(msgs); + + final long beginTime = System.currentTimeMillis(); + + log.info("consumeMessageDirectly receive new messge: {}", msg); + + try { + ConsumeConcurrentlyStatus status = this.messageListener.consumeMessage(msgs, context); + if (status != null) { + switch (status) { + case CONSUME_SUCCESS: + result.setConsumeResult(CMResult.CR_SUCCESS); + break; + case RECONSUME_LATER: + result.setConsumeResult(CMResult.CR_LATER); + break; + default: + break; + } + } + else { + result.setConsumeResult(CMResult.CR_RETURN_NULL); + } + } + catch (Throwable e) { + result.setConsumeResult(CMResult.CR_THROW_EXCEPTION); + result.setRemark(RemotingHelper.exceptionSimpleDesc(e)); + + log.warn(String.format("consumeMessageDirectly exception: %s Group: %s Msgs: %s MQ: %s",// + RemotingHelper.exceptionSimpleDesc(e),// + ConsumeMessageConcurrentlyService.this.consumerGroup,// + msgs,// + mq), e); + } + + result.setSpentTimeMills(System.currentTimeMillis() - beginTime); + + log.info("consumeMessageDirectly Result: {}", result); + + return result; + } + + + public void resetRetryTopic(final List msgs) { + final String groupTopic = MixAll.getRetryTopic(consumerGroup); + for (MessageExt msg : msgs) { + String retryTopic = msg.getProperty(MessageConst.PROPERTY_RETRY_TOPIC); + if (retryTopic != null && groupTopic.equals(msg.getTopic())) { + msg.setTopic(retryTopic); + } + } + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/ConsumeMessageOrderlyService.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/ConsumeMessageOrderlyService.java new file mode 100644 index 000000000..72da36c22 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/ConsumeMessageOrderlyService.java @@ -0,0 +1,551 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.impl.consumer; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +import org.slf4j.Logger; + +import com.alibaba.rocketmq.client.consumer.DefaultMQPushConsumer; +import com.alibaba.rocketmq.client.consumer.listener.ConsumeOrderlyContext; +import com.alibaba.rocketmq.client.consumer.listener.ConsumeOrderlyStatus; +import com.alibaba.rocketmq.client.consumer.listener.MessageListenerOrderly; +import com.alibaba.rocketmq.client.hook.ConsumeMessageContext; +import com.alibaba.rocketmq.client.log.ClientLogger; +import com.alibaba.rocketmq.client.stat.ConsumerStatsManager; +import com.alibaba.rocketmq.common.ThreadFactoryImpl; +import com.alibaba.rocketmq.common.message.MessageExt; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.common.protocol.body.CMResult; +import com.alibaba.rocketmq.common.protocol.body.ConsumeMessageDirectlyResult; +import com.alibaba.rocketmq.common.protocol.heartbeat.MessageModel; +import com.alibaba.rocketmq.remoting.common.RemotingHelper; + + +/** + * 顺序消费消息服务 + * + * @author shijia.wxr + * @since 2013-6-27 + */ +public class ConsumeMessageOrderlyService implements ConsumeMessageService { + private static final Logger log = ClientLogger.getLog(); + private final static long MaxTimeConsumeContinuously = Long.parseLong(System.getProperty( + "rocketmq.client.maxTimeConsumeContinuously", "60000")); + + private volatile boolean stoped = false; + + private final DefaultMQPushConsumerImpl defaultMQPushConsumerImpl; + private final DefaultMQPushConsumer defaultMQPushConsumer; + private final MessageListenerOrderly messageListener; + private final BlockingQueue consumeRequestQueue; + private final ThreadPoolExecutor consumeExecutor; + private final String consumerGroup; + private final MessageQueueLock messageQueueLock = new MessageQueueLock(); + + // 定时线程 + private final ScheduledExecutorService scheduledExecutorService; + + + public ConsumeMessageOrderlyService(DefaultMQPushConsumerImpl defaultMQPushConsumerImpl, + MessageListenerOrderly messageListener) { + this.defaultMQPushConsumerImpl = defaultMQPushConsumerImpl; + this.messageListener = messageListener; + + this.defaultMQPushConsumer = this.defaultMQPushConsumerImpl.getDefaultMQPushConsumer(); + this.consumerGroup = this.defaultMQPushConsumer.getConsumerGroup(); + this.consumeRequestQueue = new LinkedBlockingQueue(); + + this.consumeExecutor = new ThreadPoolExecutor(// + this.defaultMQPushConsumer.getConsumeThreadMin(),// + this.defaultMQPushConsumer.getConsumeThreadMax(),// + 1000 * 60,// + TimeUnit.MILLISECONDS,// + this.consumeRequestQueue,// + new ThreadFactoryImpl("ConsumeMessageThread_")); + + this.scheduledExecutorService = + Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl( + "ConsumeMessageScheduledThread_")); + } + + + public void start() { + // 启动定时lock队列服务 + if (MessageModel.CLUSTERING.equals(ConsumeMessageOrderlyService.this.defaultMQPushConsumerImpl + .messageModel())) { + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + ConsumeMessageOrderlyService.this.lockMQPeriodically(); + } + }, 1000 * 1, ProcessQueue.RebalanceLockInterval, TimeUnit.MILLISECONDS); + } + } + + + public void shutdown() { + this.stoped = true; + this.scheduledExecutorService.shutdown(); + this.consumeExecutor.shutdown(); + if (MessageModel.CLUSTERING.equals(this.defaultMQPushConsumerImpl.messageModel())) { + this.unlockAllMQ(); + } + } + + + public synchronized void unlockAllMQ() { + this.defaultMQPushConsumerImpl.getRebalanceImpl().unlockAll(false); + } + + + public synchronized void lockMQPeriodically() { + if (!this.stoped) { + this.defaultMQPushConsumerImpl.getRebalanceImpl().lockAll(); + } + } + + + public synchronized boolean lockOneMQ(final MessageQueue mq) { + if (!this.stoped) { + return this.defaultMQPushConsumerImpl.getRebalanceImpl().lock(mq); + } + + return false; + } + + + public void tryLockLaterAndReconsume(final MessageQueue mq, final ProcessQueue processQueue, + final long delayMills) { + this.scheduledExecutorService.schedule(new Runnable() { + @Override + public void run() { + boolean lockOK = ConsumeMessageOrderlyService.this.lockOneMQ(mq); + if (lockOK) { + ConsumeMessageOrderlyService.this.submitConsumeRequestLater(processQueue, mq, 10); + } + else { + ConsumeMessageOrderlyService.this.submitConsumeRequestLater(processQueue, mq, 3000); + } + } + }, delayMills, TimeUnit.MILLISECONDS); + } + + + public ConsumerStatsManager getConsumerStatsManager() { + return this.defaultMQPushConsumerImpl.getConsumerStatsManager(); + } + + class ConsumeRequest implements Runnable { + private final ProcessQueue processQueue; + private final MessageQueue messageQueue; + + + public ConsumeRequest(ProcessQueue processQueue, MessageQueue messageQueue) { + this.processQueue = processQueue; + this.messageQueue = messageQueue; + } + + + @Override + public void run() { + if (this.processQueue.isDroped()) { + log.warn("run, the message queue not be able to consume, because it's dropped. {}", + this.messageQueue); + return; + } + + // 保证在当前Consumer内，同一队列串行消费 + final Object objLock = messageQueueLock.fetchLockObject(this.messageQueue); + synchronized (objLock) { + // 保证在Consumer集群，同一队列串行消费 + if (MessageModel.BROADCASTING + .equals(ConsumeMessageOrderlyService.this.defaultMQPushConsumerImpl.messageModel()) + || (this.processQueue.isLocked() && !this.processQueue.isLockExpired())) { + final long beginTime = System.currentTimeMillis(); + for (boolean continueConsume = true; continueConsume;) { + if (this.processQueue.isDroped()) { + log.warn("the message queue not be able to consume, because it's dropped. {}", + this.messageQueue); + break; + } + + if (MessageModel.CLUSTERING + .equals(ConsumeMessageOrderlyService.this.defaultMQPushConsumerImpl + .messageModel()) + && !this.processQueue.isLocked()) { + log.warn("the message queue not locked, so consume later, {}", this.messageQueue); + ConsumeMessageOrderlyService.this.tryLockLaterAndReconsume(this.messageQueue, + this.processQueue, 10); + break; + } + + if (MessageModel.CLUSTERING + .equals(ConsumeMessageOrderlyService.this.defaultMQPushConsumerImpl + .messageModel()) + && this.processQueue.isLockExpired()) { + log.warn("the message queue lock expired, so consume later, {}", + this.messageQueue); + ConsumeMessageOrderlyService.this.tryLockLaterAndReconsume(this.messageQueue, + this.processQueue, 10); + break; + } + + // 在线程数小于队列数情况下，防止个别队列被饿死 + long interval = System.currentTimeMillis() - beginTime; + if (interval > MaxTimeConsumeContinuously) { + // 过10ms后再消费 + ConsumeMessageOrderlyService.this.submitConsumeRequestLater(processQueue, + messageQueue, 10); + break; + } + + final int consumeBatchSize = + ConsumeMessageOrderlyService.this.defaultMQPushConsumer + .getConsumeMessageBatchMaxSize(); + + List msgs = this.processQueue.takeMessags(consumeBatchSize); + if (!msgs.isEmpty()) { + final ConsumeOrderlyContext context = + new ConsumeOrderlyContext(this.messageQueue); + + ConsumeOrderlyStatus status = null; + + // 执行Hook + ConsumeMessageContext consumeMessageContext = null; + if (ConsumeMessageOrderlyService.this.defaultMQPushConsumerImpl.hasHook()) { + consumeMessageContext = new ConsumeMessageContext(); + consumeMessageContext + .setConsumerGroup(ConsumeMessageOrderlyService.this.defaultMQPushConsumer + .getConsumerGroup()); + consumeMessageContext.setMq(messageQueue); + consumeMessageContext.setMsgList(msgs); + consumeMessageContext.setSuccess(false); + ConsumeMessageOrderlyService.this.defaultMQPushConsumerImpl + .executeHookBefore(consumeMessageContext); + } + + long beginTimestamp = System.currentTimeMillis(); + + try { + this.processQueue.getLockConsume().lock(); + if (this.processQueue.isDroped()) { + log.warn( + "consumeMessage, the message queue not be able to consume, because it's dropped. {}", + this.messageQueue); + break; + } + + status = + messageListener.consumeMessage(Collections.unmodifiableList(msgs), + context); + } + catch (Throwable e) { + log.warn("consumeMessage exception: {} Group: {} Msgs: {} MQ: {}",// + RemotingHelper.exceptionSimpleDesc(e),// + ConsumeMessageOrderlyService.this.consumerGroup,// + msgs,// + messageQueue); + } + finally { + this.processQueue.getLockConsume().unlock(); + } + + // 针对异常返回代码打印日志 + if (null == status // + || ConsumeOrderlyStatus.ROLLBACK == status// + || ConsumeOrderlyStatus.SUSPEND_CURRENT_QUEUE_A_MOMENT == status) { + log.warn("consumeMessage Orderly return not OK, Group: {} Msgs: {} MQ: {}",// + ConsumeMessageOrderlyService.this.consumerGroup,// + msgs,// + messageQueue); + } + + long consumeRT = System.currentTimeMillis() - beginTimestamp; + + // 用户抛出异常或者返回null，都挂起队列 + if (null == status) { + status = ConsumeOrderlyStatus.SUSPEND_CURRENT_QUEUE_A_MOMENT; + } + + // 执行Hook + if (ConsumeMessageOrderlyService.this.defaultMQPushConsumerImpl.hasHook()) { + consumeMessageContext.setStatus(status.toString()); + consumeMessageContext.setSuccess(ConsumeOrderlyStatus.SUCCESS == status + || ConsumeOrderlyStatus.COMMIT == status); + ConsumeMessageOrderlyService.this.defaultMQPushConsumerImpl + .executeHookAfter(consumeMessageContext); + } + + // 记录统计信息 + ConsumeMessageOrderlyService.this.getConsumerStatsManager().incConsumeRT( + ConsumeMessageOrderlyService.this.consumerGroup, messageQueue.getTopic(), + consumeRT); + + continueConsume = + ConsumeMessageOrderlyService.this.processConsumeResult(msgs, status, + context, this); + } + else { + continueConsume = false; + } + } + } + // 没有拿到当前队列的锁，稍后再消费 + else { + if (this.processQueue.isDroped()) { + log.warn("the message queue not be able to consume, because it's dropped. {}", + this.messageQueue); + return; + } + + ConsumeMessageOrderlyService.this.tryLockLaterAndReconsume(this.messageQueue, + this.processQueue, 100); + } + } + } + + + public ProcessQueue getProcessQueue() { + return processQueue; + } + + + public MessageQueue getMessageQueue() { + return messageQueue; + } + } + + + public boolean processConsumeResult(// + final List msgs, // + final ConsumeOrderlyStatus status, // + final ConsumeOrderlyContext context, // + final ConsumeRequest consumeRequest// + ) { + boolean continueConsume = true; + long commitOffset = -1L; + // 非事务方式，自动提交 + if (context.isAutoCommit()) { + switch (status) { + case COMMIT: + case ROLLBACK: + log.warn( + "the message queue consume result is illegal, we think you want to ack these message {}", + consumeRequest.getMessageQueue()); + case SUCCESS: + commitOffset = consumeRequest.getProcessQueue().commit(); + // 统计信息 + this.getConsumerStatsManager().incConsumeOKTPS(consumerGroup, + consumeRequest.getMessageQueue().getTopic(), msgs.size()); + break; + case SUSPEND_CURRENT_QUEUE_A_MOMENT: + consumeRequest.getProcessQueue().makeMessageToCosumeAgain(msgs); + this.submitConsumeRequestLater(// + consumeRequest.getProcessQueue(), // + consumeRequest.getMessageQueue(), // + context.getSuspendCurrentQueueTimeMillis()); + continueConsume = false; + + // 统计信息 + this.getConsumerStatsManager().incConsumeFailedTPS(consumerGroup, + consumeRequest.getMessageQueue().getTopic(), msgs.size()); + break; + default: + break; + } + } + // 事务方式，由用户来控制提交回滚 + else { + switch (status) { + case SUCCESS: + // 统计信息 + this.getConsumerStatsManager().incConsumeOKTPS(consumerGroup, + consumeRequest.getMessageQueue().getTopic(), msgs.size()); + break; + case COMMIT: + commitOffset = consumeRequest.getProcessQueue().commit(); + break; + case ROLLBACK: + // 如果Rollback后，最好suspend一会儿再消费，防止应用无限Rollback下去 + consumeRequest.getProcessQueue().rollback(); + this.submitConsumeRequestLater(// + consumeRequest.getProcessQueue(), // + consumeRequest.getMessageQueue(), // + context.getSuspendCurrentQueueTimeMillis()); + continueConsume = false; + break; + case SUSPEND_CURRENT_QUEUE_A_MOMENT: + consumeRequest.getProcessQueue().makeMessageToCosumeAgain(msgs); + this.submitConsumeRequestLater(// + consumeRequest.getProcessQueue(), // + consumeRequest.getMessageQueue(), // + context.getSuspendCurrentQueueTimeMillis()); + continueConsume = false; + // 统计信息 + this.getConsumerStatsManager().incConsumeFailedTPS(consumerGroup, + consumeRequest.getMessageQueue().getTopic(), msgs.size()); + break; + default: + break; + } + } + + if (commitOffset >= 0) { + this.defaultMQPushConsumerImpl.getOffsetStore().updateOffset(consumeRequest.getMessageQueue(), + commitOffset, false); + } + + return continueConsume; + } + + + /** + * 在Consumer本地定时线程中定时重试 + */ + private void submitConsumeRequestLater(// + final ProcessQueue processQueue, // + final MessageQueue messageQueue,// + final long suspendTimeMillis// + ) { + long timeMillis = suspendTimeMillis; + if (timeMillis < 10) { + timeMillis = 10; + } + else if (timeMillis > 30000) { + timeMillis = 30000; + } + + this.scheduledExecutorService.schedule(new Runnable() { + + @Override + public void run() { + ConsumeMessageOrderlyService.this + .submitConsumeRequest(null, processQueue, messageQueue, true); + } + }, timeMillis, TimeUnit.MILLISECONDS); + } + + + @Override + public void submitConsumeRequest(// + final List msgs, // + final ProcessQueue processQueue, // + final MessageQueue messageQueue, // + final boolean dispathToConsume) { + if (dispathToConsume) { + ConsumeRequest consumeRequest = new ConsumeRequest(processQueue, messageQueue); + this.consumeExecutor.submit(consumeRequest); + } + } + + + @Override + public void updateCorePoolSize(int corePoolSize) { + if (corePoolSize > 0 // + && corePoolSize <= Short.MAX_VALUE // + && corePoolSize < this.defaultMQPushConsumer.getConsumeThreadMax()) { + this.consumeExecutor.setCorePoolSize(corePoolSize); + } + } + + + @Override + public void incCorePoolSize() { + } + + + @Override + public void decCorePoolSize() { + } + + + @Override + public int getCorePoolSize() { + return this.consumeExecutor.getCorePoolSize(); + } + + + @Override + public ConsumeMessageDirectlyResult consumeMessageDirectly(MessageExt msg, String brokerName) { + ConsumeMessageDirectlyResult result = new ConsumeMessageDirectlyResult(); + result.setOrder(true); + + List msgs = new ArrayList(); + msgs.add(msg); + MessageQueue mq = new MessageQueue(); + mq.setBrokerName(brokerName); + mq.setTopic(msg.getTopic()); + mq.setQueueId(msg.getQueueId()); + + ConsumeOrderlyContext context = new ConsumeOrderlyContext(mq); + + final long beginTime = System.currentTimeMillis(); + + log.info("consumeMessageDirectly receive new messge: {}", msg); + + try { + ConsumeOrderlyStatus status = this.messageListener.consumeMessage(msgs, context); + if (status != null) { + switch (status) { + case COMMIT: + result.setConsumeResult(CMResult.CR_COMMIT); + break; + case ROLLBACK: + result.setConsumeResult(CMResult.CR_ROLLBACK); + break; + case SUCCESS: + result.setConsumeResult(CMResult.CR_SUCCESS); + break; + case SUSPEND_CURRENT_QUEUE_A_MOMENT: + result.setConsumeResult(CMResult.CR_LATER); + break; + default: + break; + } + } + else { + result.setConsumeResult(CMResult.CR_RETURN_NULL); + } + } + catch (Throwable e) { + result.setConsumeResult(CMResult.CR_THROW_EXCEPTION); + result.setRemark(RemotingHelper.exceptionSimpleDesc(e)); + + log.warn(String.format("consumeMessageDirectly exception: %s Group: %s Msgs: %s MQ: %s",// + RemotingHelper.exceptionSimpleDesc(e),// + ConsumeMessageOrderlyService.this.consumerGroup,// + msgs,// + mq), e); + } + + result.setAutoCommit(context.isAutoCommit()); + result.setSpentTimeMills(System.currentTimeMillis() - beginTime); + + log.info("consumeMessageDirectly Result: {}", result); + + return result; + } + +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/ConsumeMessageService.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/ConsumeMessageService.java new file mode 100644 index 000000000..c9cf52d71 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/ConsumeMessageService.java @@ -0,0 +1,58 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.impl.consumer; + +import java.util.List; + +import com.alibaba.rocketmq.common.message.MessageExt; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.common.protocol.body.ConsumeMessageDirectlyResult; + + +/** + * 消费消息服务，公共接口 + * + * @author shijia.wxr + * @since 2013-7-24 + */ +public interface ConsumeMessageService { + public void start(); + + + public void shutdown(); + + + public void updateCorePoolSize(int corePoolSize); + + + public void incCorePoolSize(); + + + public void decCorePoolSize(); + + + public int getCorePoolSize(); + + + public ConsumeMessageDirectlyResult consumeMessageDirectly(final MessageExt msg, final String brokerName); + + + public void submitConsumeRequest(// + final List msgs, // + final ProcessQueue processQueue, // + final MessageQueue messageQueue, // + final boolean dispathToConsume); +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java new file mode 100644 index 000000000..78a7ea481 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/DefaultMQPullConsumerImpl.java @@ -0,0 +1,718 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.impl.consumer; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +import org.slf4j.Logger; + +import com.alibaba.rocketmq.client.QueryResult; +import com.alibaba.rocketmq.client.Validators; +import com.alibaba.rocketmq.client.consumer.DefaultMQPullConsumer; +import com.alibaba.rocketmq.client.consumer.PullCallback; +import com.alibaba.rocketmq.client.consumer.PullResult; +import com.alibaba.rocketmq.client.consumer.store.LocalFileOffsetStore; +import com.alibaba.rocketmq.client.consumer.store.OffsetStore; +import com.alibaba.rocketmq.client.consumer.store.ReadOffsetType; +import com.alibaba.rocketmq.client.consumer.store.RemoteBrokerOffsetStore; +import com.alibaba.rocketmq.client.exception.MQBrokerException; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.client.hook.FilterMessageHook; +import com.alibaba.rocketmq.client.impl.CommunicationMode; +import com.alibaba.rocketmq.client.impl.MQClientManager; +import com.alibaba.rocketmq.client.impl.factory.MQClientInstance; +import com.alibaba.rocketmq.client.log.ClientLogger; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.ServiceState; +import com.alibaba.rocketmq.common.consumer.ConsumeFromWhere; +import com.alibaba.rocketmq.common.filter.FilterAPI; +import com.alibaba.rocketmq.common.help.FAQUrl; +import com.alibaba.rocketmq.common.message.Message; +import com.alibaba.rocketmq.common.message.MessageAccessor; +import com.alibaba.rocketmq.common.message.MessageConst; +import com.alibaba.rocketmq.common.message.MessageExt; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.common.protocol.body.ConsumerRunningInfo; +import com.alibaba.rocketmq.common.protocol.heartbeat.ConsumeType; +import com.alibaba.rocketmq.common.protocol.heartbeat.MessageModel; +import com.alibaba.rocketmq.common.protocol.heartbeat.SubscriptionData; +import com.alibaba.rocketmq.common.sysflag.PullSysFlag; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.remoting.common.RemotingHelper; +import com.alibaba.rocketmq.remoting.exception.RemotingException; + + +/** + * Pull方式的Consumer实现 + * + * @author shijia.wxr + * @since 2013-7-24 + */ +public class DefaultMQPullConsumerImpl implements MQConsumerInner { + private final Logger log = ClientLogger.getLog(); + private final DefaultMQPullConsumer defaultMQPullConsumer; + private ServiceState serviceState = ServiceState.CREATE_JUST; + private MQClientInstance mQClientFactory; + private PullAPIWrapper pullAPIWrapper; + // 消费进度存储 + private OffsetStore offsetStore; + // Rebalance实现 + private RebalanceImpl rebalanceImpl = new RebalancePullImpl(this); + + // Consumer启动时间 + private final long consumerStartTimestamp = System.currentTimeMillis(); + + private final RPCHook rpcHook; + + + public DefaultMQPullConsumerImpl(final DefaultMQPullConsumer defaultMQPullConsumer, final RPCHook rpcHook) { + this.defaultMQPullConsumer = defaultMQPullConsumer; + this.rpcHook = rpcHook; + } + + + public void createTopic(String key, String newTopic, int queueNum) throws MQClientException { + createTopic(key, newTopic, queueNum, 0); + } + + + public void createTopic(String key, String newTopic, int queueNum, int topicSysFlag) + throws MQClientException { + this.makeSureStateOK(); + this.mQClientFactory.getMQAdminImpl().createTopic(key, newTopic, queueNum, topicSysFlag); + } + + + private void makeSureStateOK() throws MQClientException { + if (this.serviceState != ServiceState.RUNNING) { + throw new MQClientException("The consumer service state not OK, "// + + this.serviceState// + + FAQUrl.suggestTodo(FAQUrl.CLIENT_SERVICE_NOT_OK), null); + } + } + + + public long fetchConsumeOffset(MessageQueue mq, boolean fromStore) throws MQClientException { + this.makeSureStateOK(); + return this.offsetStore.readOffset(mq, fromStore ? ReadOffsetType.READ_FROM_STORE + : ReadOffsetType.MEMORY_FIRST_THEN_STORE); + } + + + public Set fetchMessageQueuesInBalance(String topic) throws MQClientException { + this.makeSureStateOK(); + if (null == topic) { + throw new IllegalArgumentException("topic is null"); + } + + ConcurrentHashMap mqTable = this.rebalanceImpl.getProcessQueueTable(); + Set mqResult = new HashSet(); + for (MessageQueue mq : mqTable.keySet()) { + if (mq.getTopic().equals(topic)) { + mqResult.add(mq); + } + } + + return mqResult; + } + + + public List fetchPublishMessageQueues(String topic) throws MQClientException { + this.makeSureStateOK(); + return this.mQClientFactory.getMQAdminImpl().fetchPublishMessageQueues(topic); + } + + + public Set fetchSubscribeMessageQueues(String topic) throws MQClientException { + this.makeSureStateOK(); + return this.mQClientFactory.getMQAdminImpl().fetchSubscribeMessageQueues(topic); + } + + + public long earliestMsgStoreTime(MessageQueue mq) throws MQClientException { + this.makeSureStateOK(); + return this.mQClientFactory.getMQAdminImpl().earliestMsgStoreTime(mq); + } + + + @Override + public String groupName() { + return this.defaultMQPullConsumer.getConsumerGroup(); + } + + + @Override + public MessageModel messageModel() { + return this.defaultMQPullConsumer.getMessageModel(); + } + + + @Override + public ConsumeType consumeType() { + return ConsumeType.CONSUME_ACTIVELY; + } + + + @Override + public ConsumeFromWhere consumeFromWhere() { + return ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET; + } + + + @Override + public Set subscriptions() { + Set result = new HashSet(); + + Set topics = this.defaultMQPullConsumer.getRegisterTopics(); + if (topics != null) { + synchronized (topics) { + for (String t : topics) { + SubscriptionData ms = null; + try { + ms = FilterAPI.buildSubscriptionData(this.groupName(), t, SubscriptionData.SUB_ALL); + } + catch (Exception e) { + log.error("parse subscription error", e); + } + ms.setSubVersion(0L); + result.add(ms); + } + } + } + + return result; + } + + + @Override + public void doRebalance() { + if (this.rebalanceImpl != null) { + this.rebalanceImpl.doRebalance(); + } + } + + + @Override + public void persistConsumerOffset() { + try { + this.makeSureStateOK(); + Set mqs = new HashSet(); + Set allocateMq = this.rebalanceImpl.getProcessQueueTable().keySet(); + if (allocateMq != null) { + mqs.addAll(allocateMq); + } + this.offsetStore.persistAll(mqs); + } + catch (Exception e) { + log.error("group: " + this.defaultMQPullConsumer.getConsumerGroup() + + " persistConsumerOffset exception", e); + } + } + + + @Override + public void updateTopicSubscribeInfo(String topic, Set info) { + Map subTable = this.rebalanceImpl.getSubscriptionInner(); + if (subTable != null) { + if (subTable.containsKey(topic)) { + this.rebalanceImpl.getTopicSubscribeInfoTable().put(topic, info); + } + } + } + + + @Override + public boolean isSubscribeTopicNeedUpdate(String topic) { + Map subTable = this.rebalanceImpl.getSubscriptionInner(); + if (subTable != null) { + if (subTable.containsKey(topic)) { + return !this.rebalanceImpl.topicSubscribeInfoTable.containsKey(topic); + } + } + + return false; + } + + + @Override + public boolean isUnitMode() { + return this.defaultMQPullConsumer.isUnitMode(); + } + + + public long maxOffset(MessageQueue mq) throws MQClientException { + this.makeSureStateOK(); + return this.mQClientFactory.getMQAdminImpl().maxOffset(mq); + } + + + public long minOffset(MessageQueue mq) throws MQClientException { + this.makeSureStateOK(); + return this.mQClientFactory.getMQAdminImpl().minOffset(mq); + } + + + public PullResult pull(MessageQueue mq, String subExpression, long offset, int maxNums) + throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + return this.pullSyncImpl(mq, subExpression, offset, maxNums, false); + } + + + private PullResult pullSyncImpl(MessageQueue mq, String subExpression, long offset, int maxNums, + boolean block) throws MQClientException, RemotingException, MQBrokerException, + InterruptedException { + this.makeSureStateOK(); + + if (null == mq) { + throw new MQClientException("mq is null", null); + } + + if (offset < 0) { + throw new MQClientException("offset < 0", null); + } + + if (maxNums <= 0) { + throw new MQClientException("maxNums <= 0", null); + } + + // 自动订阅 + this.subscriptionAutomatically(mq.getTopic()); + + int sysFlag = PullSysFlag.buildSysFlag(false, block, true, false); + + SubscriptionData subscriptionData; + try { + subscriptionData = FilterAPI.buildSubscriptionData(this.defaultMQPullConsumer.getConsumerGroup(),// + mq.getTopic(), subExpression); + } + catch (Exception e) { + throw new MQClientException("parse subscription error", e); + } + + long timeoutMillis = + block ? this.defaultMQPullConsumer.getConsumerTimeoutMillisWhenSuspend() + : this.defaultMQPullConsumer.getConsumerPullTimeoutMillis(); + + PullResult pullResult = this.pullAPIWrapper.pullKernelImpl(// + mq, // 1 + subscriptionData.getSubString(), // 2 + 0L, // 3 + offset, // 4 + maxNums, // 5 + sysFlag, // 6 + 0, // 7 + this.defaultMQPullConsumer.getBrokerSuspendMaxTimeMillis(), // 8 + timeoutMillis, // 9 + CommunicationMode.SYNC, // 10 + null// 11 + ); + + return this.pullAPIWrapper.processPullResult(mq, pullResult, subscriptionData); + } + + + private void subscriptionAutomatically(final String topic) { + if (!this.rebalanceImpl.getSubscriptionInner().containsKey(topic)) { + try { + SubscriptionData subscriptionData = + FilterAPI.buildSubscriptionData(this.defaultMQPullConsumer.getConsumerGroup(),// + topic, SubscriptionData.SUB_ALL); + this.rebalanceImpl.subscriptionInner.putIfAbsent(topic, subscriptionData); + } + catch (Exception e) { + } + } + } + + + public void pull(MessageQueue mq, String subExpression, long offset, int maxNums, + PullCallback pullCallback) throws MQClientException, RemotingException, InterruptedException { + this.pullAsyncImpl(mq, subExpression, offset, maxNums, pullCallback, false); + } + + + private void pullAsyncImpl(// + final MessageQueue mq,// + final String subExpression,// + final long offset,// + final int maxNums,// + final PullCallback pullCallback,// + final boolean block// + ) throws MQClientException, RemotingException, InterruptedException { + this.makeSureStateOK(); + + if (null == mq) { + throw new MQClientException("mq is null", null); + } + + if (offset < 0) { + throw new MQClientException("offset < 0", null); + } + + if (maxNums <= 0) { + throw new MQClientException("maxNums <= 0", null); + } + + if (null == pullCallback) { + throw new MQClientException("pullCallback is null", null); + } + + // 自动订阅 + this.subscriptionAutomatically(mq.getTopic()); + + try { + int sysFlag = PullSysFlag.buildSysFlag(false, block, true, false); + + final SubscriptionData subscriptionData; + try { + subscriptionData = + FilterAPI.buildSubscriptionData(this.defaultMQPullConsumer.getConsumerGroup(),// + mq.getTopic(), subExpression); + } + catch (Exception e) { + throw new MQClientException("parse subscription error", e); + } + + long timeoutMillis = + block ? this.defaultMQPullConsumer.getConsumerTimeoutMillisWhenSuspend() + : this.defaultMQPullConsumer.getConsumerPullTimeoutMillis(); + + this.pullAPIWrapper.pullKernelImpl(// + mq, // 1 + subscriptionData.getSubString(), // 2 + 0L, // 3 + offset, // 4 + maxNums, // 5 + sysFlag, // 6 + 0, // 7 + this.defaultMQPullConsumer.getBrokerSuspendMaxTimeMillis(), // 8 + timeoutMillis, // 9 + CommunicationMode.ASYNC, // 10 + new PullCallback() { + + @Override + public void onException(Throwable e) { + pullCallback.onException(e); + } + + + @Override + public void onSuccess(PullResult pullResult) { + pullCallback.onSuccess(DefaultMQPullConsumerImpl.this.pullAPIWrapper + .processPullResult(mq, pullResult, subscriptionData)); + } + }); + } + catch (MQBrokerException e) { + throw new MQClientException("pullAsync unknow exception", e); + } + } + + + public PullResult pullBlockIfNotFound(MessageQueue mq, String subExpression, long offset, int maxNums) + throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + return this.pullSyncImpl(mq, subExpression, offset, maxNums, true); + } + + + public void pullBlockIfNotFound(MessageQueue mq, String subExpression, long offset, int maxNums, + PullCallback pullCallback) throws MQClientException, RemotingException, InterruptedException { + this.pullAsyncImpl(mq, subExpression, offset, maxNums, pullCallback, true); + } + + + public QueryResult queryMessage(String topic, String key, int maxNum, long begin, long end) + throws MQClientException, InterruptedException { + this.makeSureStateOK(); + return this.mQClientFactory.getMQAdminImpl().queryMessage(topic, key, maxNum, begin, end); + } + + + public long searchOffset(MessageQueue mq, long timestamp) throws MQClientException { + this.makeSureStateOK(); + return this.mQClientFactory.getMQAdminImpl().searchOffset(mq, timestamp); + } + + + public void sendMessageBack(MessageExt msg, int delayLevel, final String brokerName) + throws RemotingException, MQBrokerException, InterruptedException, MQClientException { + try { + String brokerAddr = (null != brokerName) ? // + this.mQClientFactory.findBrokerAddressInPublish(brokerName) // + : // + RemotingHelper.parseSocketAddressAddr(msg.getStoreHost()); + + this.mQClientFactory.getMQClientAPIImpl().consumerSendMessageBack(brokerAddr, msg, + this.defaultMQPullConsumer.getConsumerGroup(), delayLevel, 3000); + } + catch (Exception e) { + log.error("sendMessageBack Exception, " + this.defaultMQPullConsumer.getConsumerGroup(), e); + + Message newMsg = + new Message(MixAll.getRetryTopic(this.defaultMQPullConsumer.getConsumerGroup()), + msg.getBody()); + + newMsg.setFlag(msg.getFlag()); + MessageAccessor.setProperties(newMsg, msg.getProperties()); + MessageAccessor.putProperty(newMsg, MessageConst.PROPERTY_RETRY_TOPIC, msg.getTopic()); + + this.mQClientFactory.getDefaultMQProducer().send(newMsg); + } + } + + + public void shutdown() { + switch (this.serviceState) { + case CREATE_JUST: + break; + case RUNNING: + this.persistConsumerOffset(); + this.mQClientFactory.unregisterConsumer(this.defaultMQPullConsumer.getConsumerGroup()); + this.mQClientFactory.shutdown(); + log.info("the consumer [{}] shutdown OK", this.defaultMQPullConsumer.getConsumerGroup()); + this.serviceState = ServiceState.SHUTDOWN_ALREADY; + break; + case SHUTDOWN_ALREADY: + break; + default: + break; + } + } + + + public void start() throws MQClientException { + switch (this.serviceState) { + case CREATE_JUST: + this.serviceState = ServiceState.START_FAILED; + + this.checkConfig(); + + this.copySubscription(); + + if (this.defaultMQPullConsumer.getMessageModel() == MessageModel.CLUSTERING) { + this.defaultMQPullConsumer.changeInstanceNameToPID(); + } + + this.mQClientFactory = + MQClientManager.getInstance().getAndCreateMQClientInstance(this.defaultMQPullConsumer, + this.rpcHook); + + // 初始化Rebalance变量 + this.rebalanceImpl.setConsumerGroup(this.defaultMQPullConsumer.getConsumerGroup()); + this.rebalanceImpl.setMessageModel(this.defaultMQPullConsumer.getMessageModel()); + this.rebalanceImpl.setAllocateMessageQueueStrategy(this.defaultMQPullConsumer + .getAllocateMessageQueueStrategy()); + this.rebalanceImpl.setmQClientFactory(this.mQClientFactory); + + this.pullAPIWrapper = new PullAPIWrapper(// + mQClientFactory,// + this.defaultMQPullConsumer.getConsumerGroup(), isUnitMode()); + // 每次拉消息之后，都会进行一次过滤。 + this.pullAPIWrapper.registerFilterMessageHook(filterMessageHookList); + + if (this.defaultMQPullConsumer.getOffsetStore() != null) { + this.offsetStore = this.defaultMQPullConsumer.getOffsetStore(); + } + else { + // 广播消费/集群消费 + switch (this.defaultMQPullConsumer.getMessageModel()) { + case BROADCASTING: + this.offsetStore = + new LocalFileOffsetStore(this.mQClientFactory, + this.defaultMQPullConsumer.getConsumerGroup()); + break; + case CLUSTERING: + this.offsetStore = + new RemoteBrokerOffsetStore(this.mQClientFactory, + this.defaultMQPullConsumer.getConsumerGroup()); + break; + default: + break; + } + } + + // 加载消费进度 + this.offsetStore.load(); + + boolean registerOK = + mQClientFactory.registerConsumer(this.defaultMQPullConsumer.getConsumerGroup(), this); + if (!registerOK) { + this.serviceState = ServiceState.CREATE_JUST; + + throw new MQClientException("The consumer group[" + + this.defaultMQPullConsumer.getConsumerGroup() + + "] has been created before, specify another name please." + + FAQUrl.suggestTodo(FAQUrl.GROUP_NAME_DUPLICATE_URL), null); + } + + mQClientFactory.start(); + log.info("the consumer [{}] start OK", this.defaultMQPullConsumer.getConsumerGroup()); + this.serviceState = ServiceState.RUNNING; + break; + case RUNNING: + case START_FAILED: + case SHUTDOWN_ALREADY: + throw new MQClientException("The PullConsumer service state not OK, maybe started once, "// + + this.serviceState// + + FAQUrl.suggestTodo(FAQUrl.CLIENT_SERVICE_NOT_OK), null); + default: + break; + } + } + + + private void copySubscription() throws MQClientException { + try { + // 复制用户初始设置的订阅关系 + Set registerTopics = this.defaultMQPullConsumer.getRegisterTopics(); + if (registerTopics != null) { + for (final String topic : registerTopics) { + SubscriptionData subscriptionData = + FilterAPI.buildSubscriptionData(this.defaultMQPullConsumer.getConsumerGroup(),// + topic, SubscriptionData.SUB_ALL); + this.rebalanceImpl.getSubscriptionInner().put(topic, subscriptionData); + } + } + } + catch (Exception e) { + throw new MQClientException("subscription exception", e); + } + } + + + private void checkConfig() throws MQClientException { + // check consumerGroup + Validators.checkGroup(this.defaultMQPullConsumer.getConsumerGroup()); + + // consumerGroup + if (null == this.defaultMQPullConsumer.getConsumerGroup()) { + throw new MQClientException("consumerGroup is null" // + + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), // + null); + } + + // consumerGroup + if (this.defaultMQPullConsumer.getConsumerGroup().equals(MixAll.DEFAULT_CONSUMER_GROUP)) { + throw new MQClientException("consumerGroup can not equal "// + + MixAll.DEFAULT_CONSUMER_GROUP // + + ", please specify another one."// + + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), // + null); + } + + // messageModel + if (null == this.defaultMQPullConsumer.getMessageModel()) { + throw new MQClientException("messageModel is null" // + + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), // + null); + } + + // allocateMessageQueueStrategy + if (null == this.defaultMQPullConsumer.getAllocateMessageQueueStrategy()) { + throw new MQClientException("allocateMessageQueueStrategy is null" // + + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), // + null); + } + } + + + public void updateConsumeOffset(MessageQueue mq, long offset) throws MQClientException { + this.makeSureStateOK(); + this.offsetStore.updateOffset(mq, offset, false); + } + + + public MessageExt viewMessage(String msgId) throws RemotingException, MQBrokerException, + InterruptedException, MQClientException { + this.makeSureStateOK(); + return this.mQClientFactory.getMQAdminImpl().viewMessage(msgId); + } + + // 消息过滤 hook + private final ArrayList filterMessageHookList = new ArrayList(); + + + public void registerFilterMessageHook(final FilterMessageHook hook) { + this.filterMessageHookList.add(hook); + log.info("register FilterMessageHook Hook, {}", hook.hookName()); + } + + + public DefaultMQPullConsumer getDefaultMQPullConsumer() { + return defaultMQPullConsumer; + } + + + public OffsetStore getOffsetStore() { + return offsetStore; + } + + + public void setOffsetStore(OffsetStore offsetStore) { + this.offsetStore = offsetStore; + } + + + public PullAPIWrapper getPullAPIWrapper() { + return pullAPIWrapper; + } + + + public void setPullAPIWrapper(PullAPIWrapper pullAPIWrapper) { + this.pullAPIWrapper = pullAPIWrapper; + } + + + public ServiceState getServiceState() { + return serviceState; + } + + + public void setServiceState(ServiceState serviceState) { + this.serviceState = serviceState; + } + + + @Override + public ConsumerRunningInfo consumerRunningInfo() { + ConsumerRunningInfo info = new ConsumerRunningInfo(); + + // 各种配置及运行数据 + Properties prop = MixAll.object2Properties(this.defaultMQPullConsumer); + prop.put(ConsumerRunningInfo.PROP_CONSUMER_START_TIMESTAMP, this.consumerStartTimestamp); + info.setProperties(prop); + + // 订阅关系 + info.getSubscriptionSet().addAll(this.subscriptions()); + return info; + } + + + public long getConsumerStartTimestamp() { + return consumerStartTimestamp; + } + + + public RebalanceImpl getRebalanceImpl() { + return rebalanceImpl; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java new file mode 100644 index 000000000..c25ce5267 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/DefaultMQPushConsumerImpl.java @@ -0,0 +1,1192 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.impl.consumer; + +import com.alibaba.rocketmq.client.QueryResult; +import com.alibaba.rocketmq.client.Validators; +import com.alibaba.rocketmq.client.consumer.DefaultMQPushConsumer; +import com.alibaba.rocketmq.client.consumer.PullCallback; +import com.alibaba.rocketmq.client.consumer.PullResult; +import com.alibaba.rocketmq.client.consumer.listener.MessageListener; +import com.alibaba.rocketmq.client.consumer.listener.MessageListenerConcurrently; +import com.alibaba.rocketmq.client.consumer.listener.MessageListenerOrderly; +import com.alibaba.rocketmq.client.consumer.store.LocalFileOffsetStore; +import com.alibaba.rocketmq.client.consumer.store.OffsetStore; +import com.alibaba.rocketmq.client.consumer.store.ReadOffsetType; +import com.alibaba.rocketmq.client.consumer.store.RemoteBrokerOffsetStore; +import com.alibaba.rocketmq.client.exception.MQBrokerException; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.client.hook.ConsumeMessageContext; +import com.alibaba.rocketmq.client.hook.ConsumeMessageHook; +import com.alibaba.rocketmq.client.hook.FilterMessageHook; +import com.alibaba.rocketmq.client.impl.CommunicationMode; +import com.alibaba.rocketmq.client.impl.MQClientManager; +import com.alibaba.rocketmq.client.impl.factory.MQClientInstance; +import com.alibaba.rocketmq.client.log.ClientLogger; +import com.alibaba.rocketmq.client.stat.ConsumerStatsManager; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.ServiceState; +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.consumer.ConsumeFromWhere; +import com.alibaba.rocketmq.common.filter.FilterAPI; +import com.alibaba.rocketmq.common.help.FAQUrl; +import com.alibaba.rocketmq.common.message.*; +import com.alibaba.rocketmq.common.protocol.body.ConsumeStatus; +import com.alibaba.rocketmq.common.protocol.body.ConsumerRunningInfo; +import com.alibaba.rocketmq.common.protocol.body.ProcessQueueInfo; +import com.alibaba.rocketmq.common.protocol.body.QueueTimeSpan; +import com.alibaba.rocketmq.common.protocol.heartbeat.ConsumeType; +import com.alibaba.rocketmq.common.protocol.heartbeat.MessageModel; +import com.alibaba.rocketmq.common.protocol.heartbeat.SubscriptionData; +import com.alibaba.rocketmq.common.protocol.route.BrokerData; +import com.alibaba.rocketmq.common.protocol.route.TopicRouteData; +import com.alibaba.rocketmq.common.sysflag.PullSysFlag; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.remoting.common.RemotingHelper; +import com.alibaba.rocketmq.remoting.exception.RemotingException; + +import org.slf4j.Logger; + +import java.util.*; +import java.util.Map.Entry; +import java.util.concurrent.ConcurrentHashMap; + + +/** + * Push方式的Consumer实现 + * + * @author shijia.wxr + * @since 2013-6-15 + */ +public class DefaultMQPushConsumerImpl implements MQConsumerInner { + // 拉消息异常时，延迟一段时间再拉 + private static final long PullTimeDelayMillsWhenException = 3000; + // 本地内存队列慢，流控间隔时间 + private static final long PullTimeDelayMillsWhenFlowControl = 50; + // 被挂起后，下次拉取间隔时间 + private static final long PullTimeDelayMillsWhenSuspend = 1000; + // 长轮询模式，Consumer连接在Broker挂起最长时间 + private static final long BrokerSuspendMaxTimeMillis = 1000 * 15; + // 长轮询模式，Consumer超时时间（必须要大于brokerSuspendMaxTimeMillis） + private static final long ConsumerTimeoutMillisWhenSuspend = 1000 * 30; + private final Logger log = ClientLogger.getLog(); + private final DefaultMQPushConsumer defaultMQPushConsumer; + // Rebalance实现 + private final RebalanceImpl rebalanceImpl = new RebalancePushImpl(this); + private ServiceState serviceState = ServiceState.CREATE_JUST; + private MQClientInstance mQClientFactory; + private PullAPIWrapper pullAPIWrapper; + // 是否暂停接收消息 suspend/resume + private volatile boolean pause = false; + // 是否顺序消费消息 + private boolean consumeOrderly = false; + // 消费消息监听器 + private MessageListener messageListenerInner; + // 消费进度存储 + private OffsetStore offsetStore; + // 消费消息服务 + private ConsumeMessageService consumeMessageService; + + // 消息过滤 hook + private final ArrayList filterMessageHookList = new ArrayList(); + + // Consumer启动时间 + private final long consumerStartTimestamp = System.currentTimeMillis(); + + + public void registerFilterMessageHook(final FilterMessageHook hook) { + this.filterMessageHookList.add(hook); + log.info("register FilterMessageHook Hook, {}", hook.hookName()); + } + + /** + * 消费每条消息会回调 + */ + private final ArrayList consumeMessageHookList = new ArrayList(); + + private final RPCHook rpcHook; + + + public DefaultMQPushConsumerImpl(DefaultMQPushConsumer defaultMQPushConsumer, RPCHook rpcHook) { + this.defaultMQPushConsumer = defaultMQPushConsumer; + this.rpcHook = rpcHook; + } + + + public boolean hasHook() { + return !this.consumeMessageHookList.isEmpty(); + } + + + public void registerConsumeMessageHook(final ConsumeMessageHook hook) { + this.consumeMessageHookList.add(hook); + log.info("register consumeMessageHook Hook, {}", hook.hookName()); + } + + + public void executeHookBefore(final ConsumeMessageContext context) { + if (!this.consumeMessageHookList.isEmpty()) { + for (ConsumeMessageHook hook : this.consumeMessageHookList) { + try { + hook.consumeMessageBefore(context); + } + catch (Throwable e) { + } + } + } + } + + + public void executeHookAfter(final ConsumeMessageContext context) { + if (!this.consumeMessageHookList.isEmpty()) { + for (ConsumeMessageHook hook : this.consumeMessageHookList) { + try { + hook.consumeMessageAfter(context); + } + catch (Throwable e) { + } + } + } + } + + + public void createTopic(String key, String newTopic, int queueNum) throws MQClientException { + createTopic(key, newTopic, queueNum, 0); + } + + + public void createTopic(String key, String newTopic, int queueNum, int topicSysFlag) + throws MQClientException { + this.mQClientFactory.getMQAdminImpl().createTopic(key, newTopic, queueNum, topicSysFlag); + } + + + public Set fetchSubscribeMessageQueues(String topic) throws MQClientException { + Set result = this.rebalanceImpl.getTopicSubscribeInfoTable().get(topic); + if (null == result) { + this.mQClientFactory.updateTopicRouteInfoFromNameServer(topic); + result = this.rebalanceImpl.getTopicSubscribeInfoTable().get(topic); + } + + if (null == result) { + throw new MQClientException("The topic[" + topic + "] not exist", null); + } + + return result; + } + + + public DefaultMQPushConsumer getDefaultMQPushConsumer() { + return defaultMQPushConsumer; + } + + + public long earliestMsgStoreTime(MessageQueue mq) throws MQClientException { + return this.mQClientFactory.getMQAdminImpl().earliestMsgStoreTime(mq); + } + + + public long maxOffset(MessageQueue mq) throws MQClientException { + return this.mQClientFactory.getMQAdminImpl().maxOffset(mq); + } + + + public long minOffset(MessageQueue mq) throws MQClientException { + return this.mQClientFactory.getMQAdminImpl().minOffset(mq); + } + + + public OffsetStore getOffsetStore() { + return offsetStore; + } + + + public void setOffsetStore(OffsetStore offsetStore) { + this.offsetStore = offsetStore; + } + + + @Override + public String groupName() { + return this.defaultMQPushConsumer.getConsumerGroup(); + } + + + @Override + public MessageModel messageModel() { + return this.defaultMQPushConsumer.getMessageModel(); + } + + + @Override + public ConsumeType consumeType() { + return ConsumeType.CONSUME_PASSIVELY; + } + + + @Override + public ConsumeFromWhere consumeFromWhere() { + return this.defaultMQPushConsumer.getConsumeFromWhere(); + } + + + @Override + public Set subscriptions() { + Set subSet = new HashSet(); + + subSet.addAll(this.rebalanceImpl.getSubscriptionInner().values()); + + return subSet; + } + + + @Override + public void doRebalance() { + if (this.rebalanceImpl != null) { + this.rebalanceImpl.doRebalance(); + } + } + + + @Override + public void persistConsumerOffset() { + try { + this.makeSureStateOK(); + Set mqs = new HashSet(); + Set allocateMq = this.rebalanceImpl.getProcessQueueTable().keySet(); + if (allocateMq != null) { + mqs.addAll(allocateMq); + } + + this.offsetStore.persistAll(mqs); + } + catch (Exception e) { + log.error("group: " + this.defaultMQPushConsumer.getConsumerGroup() + + " persistConsumerOffset exception", e); + } + } + + + @Override + public void updateTopicSubscribeInfo(String topic, Set info) { + Map subTable = this.getSubscriptionInner(); + if (subTable != null) { + if (subTable.containsKey(topic)) { + this.rebalanceImpl.topicSubscribeInfoTable.put(topic, info); + } + } + } + + + public ConcurrentHashMap getSubscriptionInner() { + return this.rebalanceImpl.getSubscriptionInner(); + } + + + @Override + public boolean isSubscribeTopicNeedUpdate(String topic) { + Map subTable = this.getSubscriptionInner(); + if (subTable != null) { + if (subTable.containsKey(topic)) { + return !this.rebalanceImpl.topicSubscribeInfoTable.containsKey(topic); + } + } + + return false; + } + + + /** + * 通过Tag过滤时，会存在offset不准确的情况，需要纠正 + */ + private void correctTagsOffset(final PullRequest pullRequest) { + // 说明本地没有可消费的消息 + if (0L == pullRequest.getProcessQueue().getMsgCount().get()) { + this.offsetStore.updateOffset(pullRequest.getMessageQueue(), pullRequest.getNextOffset(), true); + } + } + + private long flowControlTimes1 = 0; + private long flowControlTimes2 = 0; + + + public void pullMessage(final PullRequest pullRequest) { + final ProcessQueue processQueue = pullRequest.getProcessQueue(); + if (processQueue.isDroped()) { + log.info("the pull request[{}] is droped.", pullRequest.toString()); + return; + } + + // 标明尝试拉消息了 + pullRequest.getProcessQueue().setLastPullTimestamp(System.currentTimeMillis()); + + // 检测Consumer是否启动 + try { + this.makeSureStateOK(); + } + catch (MQClientException e) { + log.warn("pullMessage exception, consumer state not ok", e); + this.executePullRequestLater(pullRequest, PullTimeDelayMillsWhenException); + return; + } + + // 检测Consumer是否被挂起 + if (this.isPause()) { + log.warn("consumer was paused, execute pull request later. instanceName={}", + this.defaultMQPushConsumer.getInstanceName()); + this.executePullRequestLater(pullRequest, PullTimeDelayMillsWhenSuspend); + return; + } + + // 流量控制，队列中消息总数 + long size = processQueue.getMsgCount().get(); + if (size > this.defaultMQPushConsumer.getPullThresholdForQueue()) { + this.executePullRequestLater(pullRequest, PullTimeDelayMillsWhenFlowControl); + if ((flowControlTimes1++ % 1000) == 0) { + log.warn("the consumer message buffer is full, so do flow control, {} {} {}", size, + pullRequest, flowControlTimes1); + } + return; + } + + // 流量控制，队列中消息最大跨度 + if (!this.consumeOrderly) { + if (processQueue.getMaxSpan() > this.defaultMQPushConsumer.getConsumeConcurrentlyMaxSpan()) { + this.executePullRequestLater(pullRequest, PullTimeDelayMillsWhenFlowControl); + if ((flowControlTimes2++ % 1000) == 0) { + log.warn("the queue's messages, span too long, so do flow control, {} {} {}", + processQueue.getMaxSpan(), pullRequest, flowControlTimes2); + } + return; + } + } + + // 查询订阅关系 + final SubscriptionData subscriptionData = + this.rebalanceImpl.getSubscriptionInner().get(pullRequest.getMessageQueue().getTopic()); + if (null == subscriptionData) { + // 由于并发关系，即使找不到订阅关系，也要重试下，防止丢失PullRequest + this.executePullRequestLater(pullRequest, PullTimeDelayMillsWhenException); + log.warn("find the consumer's subscription failed, {}", pullRequest); + return; + } + + final long beginTimestamp = System.currentTimeMillis(); + + PullCallback pullCallback = new PullCallback() { + @Override + public void onSuccess(PullResult pullResult) { + if (pullResult != null) { + pullResult = + DefaultMQPushConsumerImpl.this.pullAPIWrapper.processPullResult( + pullRequest.getMessageQueue(), pullResult, subscriptionData); + + switch (pullResult.getPullStatus()) { + case FOUND: + long prevRequestOffset = pullRequest.getNextOffset(); + pullRequest.setNextOffset(pullResult.getNextBeginOffset()); + long pullRT = System.currentTimeMillis() - beginTimestamp; + DefaultMQPushConsumerImpl.this.getConsumerStatsManager().incPullRT( + pullRequest.getConsumerGroup(), pullRequest.getMessageQueue().getTopic(), pullRT); + + long firstMsgOffset = Long.MAX_VALUE; + if (pullResult.getMsgFoundList() == null || pullResult.getMsgFoundList().isEmpty()) { + DefaultMQPushConsumerImpl.this.executePullRequestImmediately(pullRequest); + } + else { + firstMsgOffset = pullResult.getMsgFoundList().get(0).getQueueOffset(); + + // 统计打点 + DefaultMQPushConsumerImpl.this.getConsumerStatsManager().incPullTPS( + pullRequest.getConsumerGroup(), pullRequest.getMessageQueue().getTopic(), + pullResult.getMsgFoundList().size()); + + boolean dispathToConsume = processQueue.putMessage(pullResult.getMsgFoundList()); + DefaultMQPushConsumerImpl.this.consumeMessageService.submitConsumeRequest(// + pullResult.getMsgFoundList(), // + processQueue, // + pullRequest.getMessageQueue(), // + dispathToConsume); + + // 流控 + if (DefaultMQPushConsumerImpl.this.defaultMQPushConsumer.getPullInterval() > 0) { + DefaultMQPushConsumerImpl.this.executePullRequestLater(pullRequest, + DefaultMQPushConsumerImpl.this.defaultMQPushConsumer.getPullInterval()); + } + // 立刻拉消息 + else { + DefaultMQPushConsumerImpl.this.executePullRequestImmediately(pullRequest); + } + } + + // 收到的消息Offset比请求的小，则可能服务器数据有误 + if (pullResult.getNextBeginOffset() < prevRequestOffset// + || firstMsgOffset < prevRequestOffset) { + log.warn( + "[BUG] pull message result maybe data wrong, nextBeginOffset: {} firstMsgOffset: {} prevRequestOffset: {}",// + pullResult.getNextBeginOffset(),// + firstMsgOffset,// + prevRequestOffset); + } + + break; + case NO_NEW_MSG: + pullRequest.setNextOffset(pullResult.getNextBeginOffset()); + + DefaultMQPushConsumerImpl.this.correctTagsOffset(pullRequest); + + DefaultMQPushConsumerImpl.this.executePullRequestImmediately(pullRequest); + break; + case NO_MATCHED_MSG: + pullRequest.setNextOffset(pullResult.getNextBeginOffset()); + + DefaultMQPushConsumerImpl.this.correctTagsOffset(pullRequest); + + DefaultMQPushConsumerImpl.this.executePullRequestImmediately(pullRequest); + break; + case OFFSET_ILLEGAL: + log.warn("the pull request offset illegal, {} {}",// + pullRequest.toString(), pullResult.toString()); + + pullRequest.setNextOffset(pullResult.getNextBeginOffset()); + + // 第一步、缓存队列里的消息全部废弃 + pullRequest.getProcessQueue().setDroped(true); + // 第二步、等待10s后再执行，防止Offset更新后又被覆盖 + DefaultMQPushConsumerImpl.this.executeTaskLater(new Runnable() { + + @Override + public void run() { + try { + // 第三步、纠正内部Offset + DefaultMQPushConsumerImpl.this.offsetStore.updateOffset( + pullRequest.getMessageQueue(), pullRequest.getNextOffset(), false); + + // 第四步、将最新的Offset更新到服务器 + DefaultMQPushConsumerImpl.this.offsetStore.persist(pullRequest + .getMessageQueue()); + + // 第五步、丢弃当前PullRequest，并且从Rebalabce结果里删除，等待下次Rebalance时，取纠正后的Offset + DefaultMQPushConsumerImpl.this.rebalanceImpl + .removeProcessQueue(pullRequest.getMessageQueue()); + + log.warn("fix the pull request offset, {}", pullRequest); + } + catch (Throwable e) { + log.error("executeTaskLater Exception", e); + } + } + }, 10000); + break; + default: + break; + } + } + } + + + @Override + public void onException(Throwable e) { + if (!pullRequest.getMessageQueue().getTopic().startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) { + log.warn("execute the pull request exception", e); + } + + DefaultMQPushConsumerImpl.this.executePullRequestLater(pullRequest, + PullTimeDelayMillsWhenException); + } + }; + + boolean commitOffsetEnable = false; + long commitOffsetValue = 0L; + if (MessageModel.CLUSTERING == this.defaultMQPushConsumer.getMessageModel()) { + commitOffsetValue = + this.offsetStore.readOffset(pullRequest.getMessageQueue(), + ReadOffsetType.READ_FROM_MEMORY); + if (commitOffsetValue > 0) { + commitOffsetEnable = true; + } + } + + String subExpression = null; + boolean classFilter = false; + SubscriptionData sd = + this.rebalanceImpl.getSubscriptionInner().get(pullRequest.getMessageQueue().getTopic()); + if (sd != null) { + if (this.defaultMQPushConsumer.isPostSubscriptionWhenPull() && !sd.isClassFilterMode()) { + subExpression = sd.getSubString(); + } + + classFilter = sd.isClassFilterMode(); + } + + int sysFlag = PullSysFlag.buildSysFlag(// + commitOffsetEnable, // commitOffset + true, // suspend + subExpression != null,// subscription + classFilter // class filter + ); + try { + this.pullAPIWrapper.pullKernelImpl(// + pullRequest.getMessageQueue(), // 1 + subExpression, // 2 + subscriptionData.getSubVersion(), // 3 + pullRequest.getNextOffset(), // 4 + this.defaultMQPushConsumer.getPullBatchSize(), // 5 + sysFlag, // 6 + commitOffsetValue,// 7 + BrokerSuspendMaxTimeMillis, // 8 + ConsumerTimeoutMillisWhenSuspend, // 9 + CommunicationMode.ASYNC, // 10 + pullCallback// 11 + ); + } + catch (Exception e) { + log.error("pullKernelImpl exception", e); + this.executePullRequestLater(pullRequest, PullTimeDelayMillsWhenException); + } + } + + + /** + * 立刻执行这个PullRequest + */ + public void executePullRequestImmediately(final PullRequest pullRequest) { + this.mQClientFactory.getPullMessageService().executePullRequestImmediately(pullRequest); + } + + + public void executeTaskLater(final Runnable r, final long timeDelay) { + this.mQClientFactory.getPullMessageService().executeTaskLater(r, timeDelay); + } + + + /** + * 稍后再执行这个PullRequest + */ + private void executePullRequestLater(final PullRequest pullRequest, final long timeDelay) { + this.mQClientFactory.getPullMessageService().executePullRequestLater(pullRequest, timeDelay); + } + + + public boolean isPause() { + return pause; + } + + + public void setPause(boolean pause) { + this.pause = pause; + } + + + private void makeSureStateOK() throws MQClientException { + if (this.serviceState != ServiceState.RUNNING) { + throw new MQClientException("The consumer service state not OK, "// + + this.serviceState// + + FAQUrl.suggestTodo(FAQUrl.CLIENT_SERVICE_NOT_OK), null); + } + } + + + public QueryResult queryMessage(String topic, String key, int maxNum, long begin, long end) + throws MQClientException, InterruptedException { + return this.mQClientFactory.getMQAdminImpl().queryMessage(topic, key, maxNum, begin, end); + } + + + public void registerMessageListener(MessageListener messageListener) { + this.messageListenerInner = messageListener; + } + + + public void resume() { + this.pause = false; + log.info("resume this consumer, {}", this.defaultMQPushConsumer.getConsumerGroup()); + } + + + public long searchOffset(MessageQueue mq, long timestamp) throws MQClientException { + return this.mQClientFactory.getMQAdminImpl().searchOffset(mq, timestamp); + } + + + public void sendMessageBack(MessageExt msg, int delayLevel, final String brokerName) + throws RemotingException, MQBrokerException, InterruptedException, MQClientException { + try { + String brokerAddr = (null != brokerName) ? // + this.mQClientFactory.findBrokerAddressInPublish(brokerName) // + : // + RemotingHelper.parseSocketAddressAddr(msg.getStoreHost()); + + this.mQClientFactory.getMQClientAPIImpl().consumerSendMessageBack(brokerAddr, msg, + this.defaultMQPushConsumer.getConsumerGroup(), delayLevel, 5000); + } + catch (Exception e) { + log.error("sendMessageBack Exception, " + this.defaultMQPushConsumer.getConsumerGroup(), e); + + Message newMsg = + new Message(MixAll.getRetryTopic(this.defaultMQPushConsumer.getConsumerGroup()), + msg.getBody()); + + // 保存源生消息的 msgId + String originMsgId = MessageAccessor.getOriginMessageId(msg); + MessageAccessor.setOriginMessageId(newMsg, UtilAll.isBlank(originMsgId) ? msg.getMsgId() + : originMsgId); + + newMsg.setFlag(msg.getFlag()); + // 这里要删除无用的属性，防止服务器发生冲突。TODO + MessageAccessor.setProperties(newMsg, msg.getProperties()); + MessageAccessor.putProperty(newMsg, MessageConst.PROPERTY_RETRY_TOPIC, msg.getTopic()); + int reTimes = msg.getReconsumeTimes() + 1; + MessageAccessor.setReconsumeTime(newMsg, reTimes + ""); + // 设置Delay Level + newMsg.setDelayTimeLevel(3 + reTimes); + + this.mQClientFactory.getDefaultMQProducer().send(newMsg); + } + } + + + public void shutdown() { + switch (this.serviceState) { + case CREATE_JUST: + break; + case RUNNING: + this.consumeMessageService.shutdown(); + this.persistConsumerOffset(); + this.mQClientFactory.unregisterConsumer(this.defaultMQPushConsumer.getConsumerGroup()); + this.mQClientFactory.shutdown(); + log.info("the consumer [{}] shutdown OK", this.defaultMQPushConsumer.getConsumerGroup()); + this.rebalanceImpl.destroy(); + this.serviceState = ServiceState.SHUTDOWN_ALREADY; + break; + case SHUTDOWN_ALREADY: + break; + default: + break; + } + } + + + public void start() throws MQClientException { + switch (this.serviceState) { + case CREATE_JUST: + log.info("the consumer [{}] start beginning. messageModel={}, isUnitMode={}", + this.defaultMQPushConsumer.getConsumerGroup(), this.defaultMQPushConsumer.getMessageModel(), + this.defaultMQPushConsumer.isUnitMode()); + this.serviceState = ServiceState.START_FAILED; + + this.checkConfig(); + + // 复制订阅关系 + this.copySubscription(); + + if (this.defaultMQPushConsumer.getMessageModel() == MessageModel.CLUSTERING) { + this.defaultMQPushConsumer.changeInstanceNameToPID(); + } + + this.mQClientFactory = + MQClientManager.getInstance().getAndCreateMQClientInstance(this.defaultMQPushConsumer, + this.rpcHook); + + // 初始化Rebalance变量 + this.rebalanceImpl.setConsumerGroup(this.defaultMQPushConsumer.getConsumerGroup()); + this.rebalanceImpl.setMessageModel(this.defaultMQPushConsumer.getMessageModel()); + this.rebalanceImpl.setAllocateMessageQueueStrategy(this.defaultMQPushConsumer + .getAllocateMessageQueueStrategy()); + this.rebalanceImpl.setmQClientFactory(this.mQClientFactory); + + this.pullAPIWrapper = new PullAPIWrapper(// + mQClientFactory,// + this.defaultMQPushConsumer.getConsumerGroup(), isUnitMode()); + // 每次拉消息之后，都会进行一次过滤。 + this.pullAPIWrapper.registerFilterMessageHook(filterMessageHookList); + + if (this.defaultMQPushConsumer.getOffsetStore() != null) { + this.offsetStore = this.defaultMQPushConsumer.getOffsetStore(); + } + else { + // 广播消费/集群消费 + switch (this.defaultMQPushConsumer.getMessageModel()) { + case BROADCASTING: + this.offsetStore = + new LocalFileOffsetStore(this.mQClientFactory, + this.defaultMQPushConsumer.getConsumerGroup()); + break; + case CLUSTERING: + this.offsetStore = + new RemoteBrokerOffsetStore(this.mQClientFactory, + this.defaultMQPushConsumer.getConsumerGroup()); + break; + default: + break; + } + } + // 加载消费进度 + this.offsetStore.load(); + + // 启动消费消息服务 + if (this.getMessageListenerInner() instanceof MessageListenerOrderly) { + this.consumeOrderly = true; + this.consumeMessageService = + new ConsumeMessageOrderlyService(this, + (MessageListenerOrderly) this.getMessageListenerInner()); + } + else if (this.getMessageListenerInner() instanceof MessageListenerConcurrently) { + this.consumeOrderly = false; + this.consumeMessageService = + new ConsumeMessageConcurrentlyService(this, + (MessageListenerConcurrently) this.getMessageListenerInner()); + } + + this.consumeMessageService.start(); + + boolean registerOK = + mQClientFactory.registerConsumer(this.defaultMQPushConsumer.getConsumerGroup(), this); + if (!registerOK) { + this.serviceState = ServiceState.CREATE_JUST; + this.consumeMessageService.shutdown(); + throw new MQClientException("The consumer group[" + + this.defaultMQPushConsumer.getConsumerGroup() + + "] has been created before, specify another name please." + + FAQUrl.suggestTodo(FAQUrl.GROUP_NAME_DUPLICATE_URL), null); + } + + mQClientFactory.start(); + log.info("the consumer [{}] start OK.", this.defaultMQPushConsumer.getConsumerGroup()); + this.serviceState = ServiceState.RUNNING; + break; + case RUNNING: + case START_FAILED: + case SHUTDOWN_ALREADY: + throw new MQClientException("The PushConsumer service state not OK, maybe started once, "// + + this.serviceState// + + FAQUrl.suggestTodo(FAQUrl.CLIENT_SERVICE_NOT_OK), null); + default: + break; + } + + this.updateTopicSubscribeInfoWhenSubscriptionChanged(); + + this.mQClientFactory.sendHeartbeatToAllBrokerWithLock(); + + this.mQClientFactory.rebalanceImmediately(); + } + + + private void checkConfig() throws MQClientException { + // consumerGroup 有效性检查 + Validators.checkGroup(this.defaultMQPushConsumer.getConsumerGroup()); + + // consumerGroup + if (null == this.defaultMQPushConsumer.getConsumerGroup()) { + throw new MQClientException("consumerGroup is null" // + + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), // + null); + } + + // consumerGroup + if (this.defaultMQPushConsumer.getConsumerGroup().equals(MixAll.DEFAULT_CONSUMER_GROUP)) { + throw new MQClientException("consumerGroup can not equal "// + + MixAll.DEFAULT_CONSUMER_GROUP // + + ", please specify another one."// + + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), // + null); + } + + // messageModel + if (null == this.defaultMQPushConsumer.getMessageModel()) { + throw new MQClientException("messageModel is null" // + + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), // + null); + } + + // consumeFromWhereOffset + if (null == this.defaultMQPushConsumer.getConsumeFromWhere()) { + throw new MQClientException("consumeFromWhere is null" // + + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), // + null); + } + + // 校验回溯时间戳格式是否正确 + Date dt = UtilAll.parseDate(this.defaultMQPushConsumer.getConsumeTimestamp(), UtilAll.yyyyMMddHHmmss); + if (null == dt) { + throw new MQClientException("consumeTimestamp is invalid, yyyyMMddHHmmss" // + + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), // + null); + } + + // allocateMessageQueueStrategy + if (null == this.defaultMQPushConsumer.getAllocateMessageQueueStrategy()) { + throw new MQClientException("allocateMessageQueueStrategy is null" // + + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), // + null); + } + + // subscription + if (null == this.defaultMQPushConsumer.getSubscription()) { + throw new MQClientException("subscription is null" // + + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), // + null); + } + + // messageListener + if (null == this.defaultMQPushConsumer.getMessageListener()) { + throw new MQClientException("messageListener is null" // + + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), // + null); + } + + boolean orderly = this.defaultMQPushConsumer.getMessageListener() instanceof MessageListenerOrderly; + boolean concurrently = + this.defaultMQPushConsumer.getMessageListener() instanceof MessageListenerConcurrently; + if (!orderly && !concurrently) { + throw new MQClientException( + "messageListener must be instanceof MessageListenerOrderly or MessageListenerConcurrently" // + + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), // + null); + } + + // consumeThreadMin + if (this.defaultMQPushConsumer.getConsumeThreadMin() < 1 // + || this.defaultMQPushConsumer.getConsumeThreadMin() > 1000// + || this.defaultMQPushConsumer.getConsumeThreadMin() > this.defaultMQPushConsumer + .getConsumeThreadMax()// + ) { + throw new MQClientException("consumeThreadMin Out of range [1, 1000]" // + + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), // + null); + } + + // consumeThreadMax + if (this.defaultMQPushConsumer.getConsumeThreadMax() < 1 + || this.defaultMQPushConsumer.getConsumeThreadMax() > 1000) { + throw new MQClientException("consumeThreadMax Out of range [1, 1000]" // + + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), // + null); + } + + // consumeConcurrentlyMaxSpan + if (this.defaultMQPushConsumer.getConsumeConcurrentlyMaxSpan() < 1 + || this.defaultMQPushConsumer.getConsumeConcurrentlyMaxSpan() > 65535) { + throw new MQClientException("consumeConcurrentlyMaxSpan Out of range [1, 65535]" // + + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), // + null); + } + + // pullThresholdForQueue + if (this.defaultMQPushConsumer.getPullThresholdForQueue() < 1 + || this.defaultMQPushConsumer.getPullThresholdForQueue() > 65535) { + throw new MQClientException("pullThresholdForQueue Out of range [1, 65535]" // + + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), // + null); + } + + // pullInterval + if (this.defaultMQPushConsumer.getPullInterval() < 0 + || this.defaultMQPushConsumer.getPullInterval() > 65535) { + throw new MQClientException("pullInterval Out of range [0, 65535]" // + + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), // + null); + } + + // consumeMessageBatchMaxSize + if (this.defaultMQPushConsumer.getConsumeMessageBatchMaxSize() < 1 + || this.defaultMQPushConsumer.getConsumeMessageBatchMaxSize() > 1024) { + throw new MQClientException("consumeMessageBatchMaxSize Out of range [1, 1024]" // + + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), // + null); + } + + // pullBatchSize + if (this.defaultMQPushConsumer.getPullBatchSize() < 1 + || this.defaultMQPushConsumer.getPullBatchSize() > 1024) { + throw new MQClientException("pullBatchSize Out of range [1, 1024]" // + + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), // + null); + } + } + + + private void copySubscription() throws MQClientException { + try { + // 复制用户初始设置的订阅关系 + Map sub = this.defaultMQPushConsumer.getSubscription(); + if (sub != null) { + for (final Map.Entry entry : sub.entrySet()) { + final String topic = entry.getKey(); + final String subString = entry.getValue(); + SubscriptionData subscriptionData = + FilterAPI.buildSubscriptionData(this.defaultMQPushConsumer.getConsumerGroup(),// + topic, subString); + this.rebalanceImpl.getSubscriptionInner().put(topic, subscriptionData); + } + } + + if (null == this.messageListenerInner) { + this.messageListenerInner = this.defaultMQPushConsumer.getMessageListener(); + } + + switch (this.defaultMQPushConsumer.getMessageModel()) { + case BROADCASTING: + break; + case CLUSTERING: + // 默认订阅消息重试Topic + final String retryTopic = MixAll.getRetryTopic(this.defaultMQPushConsumer.getConsumerGroup()); + SubscriptionData subscriptionData = + FilterAPI.buildSubscriptionData(this.defaultMQPushConsumer.getConsumerGroup(),// + retryTopic, SubscriptionData.SUB_ALL); + this.rebalanceImpl.getSubscriptionInner().put(retryTopic, subscriptionData); + break; + default: + break; + } + } + catch (Exception e) { + throw new MQClientException("subscription exception", e); + } + } + + + public MessageListener getMessageListenerInner() { + return messageListenerInner; + } + + + private void updateTopicSubscribeInfoWhenSubscriptionChanged() { + Map subTable = this.getSubscriptionInner(); + if (subTable != null) { + for (final Map.Entry entry : subTable.entrySet()) { + final String topic = entry.getKey(); + this.mQClientFactory.updateTopicRouteInfoFromNameServer(topic); + } + } + } + + + public void subscribe(String topic, String subExpression) throws MQClientException { + try { + SubscriptionData subscriptionData = + FilterAPI.buildSubscriptionData(this.defaultMQPushConsumer.getConsumerGroup(),// + topic, subExpression); + this.rebalanceImpl.getSubscriptionInner().put(topic, subscriptionData); + // 发送心跳，将变更的订阅关系注册上去 + if (this.mQClientFactory != null) { + this.mQClientFactory.sendHeartbeatToAllBrokerWithLock(); + } + } + catch (Exception e) { + throw new MQClientException("subscription exception", e); + } + } + + + public void suspend() { + this.pause = true; + log.info("suspend this consumer, {}", this.defaultMQPushConsumer.getConsumerGroup()); + } + + + public void unsubscribe(String topic) { + this.rebalanceImpl.getSubscriptionInner().remove(topic); + } + + + public void updateConsumeOffset(MessageQueue mq, long offset) { + this.offsetStore.updateOffset(mq, offset, false); + } + + + public void updateCorePoolSize(int corePoolSize) { + this.consumeMessageService.updateCorePoolSize(corePoolSize); + } + + + public MessageExt viewMessage(String msgId) throws RemotingException, MQBrokerException, + InterruptedException, MQClientException { + return this.mQClientFactory.getMQAdminImpl().viewMessage(msgId); + } + + + public RebalanceImpl getRebalanceImpl() { + return rebalanceImpl; + } + + + public boolean isConsumeOrderly() { + return consumeOrderly; + } + + + public void setConsumeOrderly(boolean consumeOrderly) { + this.consumeOrderly = consumeOrderly; + } + + + @Override + public boolean isUnitMode() { + return this.defaultMQPushConsumer.isUnitMode(); + } + + + public void resetOffsetByTimeStamp(long timeStamp) throws RemotingException, MQBrokerException, + InterruptedException, MQClientException { + for (String topic : rebalanceImpl.getSubscriptionInner().keySet()) { + Set mqs = rebalanceImpl.getTopicSubscribeInfoTable().get(topic); + Map offsetTable = new HashMap(); + if (mqs != null) { + for (MessageQueue mq : mqs) { + long offset = searchOffset(mq, timeStamp); + offsetTable.put(mq, offset); + } + this.mQClientFactory.resetOffset(topic, groupName(), offsetTable); + } + } + } + + + public MQClientInstance getmQClientFactory() { + return mQClientFactory; + } + + + public void setmQClientFactory(MQClientInstance mQClientFactory) { + this.mQClientFactory = mQClientFactory; + } + + + public ServiceState getServiceState() { + return serviceState; + } + + + public void setServiceState(ServiceState serviceState) { + this.serviceState = serviceState; + } + + + private long computeDuijiTotal() { + long msgDuijiCntTotal = 0; + ConcurrentHashMap processQueueTable = + this.rebalanceImpl.getProcessQueueTable(); + Iterator> it = processQueueTable.entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + ProcessQueue value = next.getValue(); + msgDuijiCntTotal += value.getMsgDuijiCnt(); + } + + return msgDuijiCntTotal; + } + + + /** + * 根据消息堆积数量，动态调整线程池数量 + */ + public void adjustThreadPool() { + long computeDuijiTotal = this.computeDuijiTotal(); + long adjustThreadPoolNumsThreshold = this.defaultMQPushConsumer.getAdjustThreadPoolNumsThreshold(); + + long incThreshold = (long) (adjustThreadPoolNumsThreshold * 1.0); + + long decThreshold = (long) (adjustThreadPoolNumsThreshold * 0.8); + + // 增加线程池线程数量 + if (computeDuijiTotal >= incThreshold) { + this.consumeMessageService.incCorePoolSize(); + } + + // 开始减少线程池线程数量 + if (computeDuijiTotal < decThreshold) { + this.consumeMessageService.decCorePoolSize(); + } + } + + + @Override + public ConsumerRunningInfo consumerRunningInfo() { + ConsumerRunningInfo info = new ConsumerRunningInfo(); + + // 各种配置及运行数据 + Properties prop = MixAll.object2Properties(this.defaultMQPushConsumer); + + prop.put(ConsumerRunningInfo.PROP_CONSUME_ORDERLY, String.valueOf(this.consumeOrderly)); + prop.put(ConsumerRunningInfo.PROP_THREADPOOL_CORE_SIZE, + String.valueOf(this.consumeMessageService.getCorePoolSize())); + prop.put(ConsumerRunningInfo.PROP_CONSUMER_START_TIMESTAMP, + String.valueOf(this.consumerStartTimestamp)); + + info.setProperties(prop); + + // 订阅关系 + Set subSet = this.subscriptions(); + info.getSubscriptionSet().addAll(subSet); + + // 消费进度、Rebalance、内部消费队列的信息 + Iterator> it = + this.rebalanceImpl.getProcessQueueTable().entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + MessageQueue mq = next.getKey(); + ProcessQueue pq = next.getValue(); + + ProcessQueueInfo pqinfo = new ProcessQueueInfo(); + pqinfo.setCommitOffset(this.offsetStore.readOffset(mq, ReadOffsetType.MEMORY_FIRST_THEN_STORE)); + pq.fillProcessQueueInfo(pqinfo); + info.getMqTable().put(mq, pqinfo); + } + + // RT、TPS统计 + for (SubscriptionData sd : subSet) { + ConsumeStatus consumeStatus = + this.mQClientFactory.getConsumerStatsManager().consumeStatus(this.groupName(), + sd.getTopic()); + info.getStatusTable().put(sd.getTopic(), consumeStatus); + } + + return info; + } + + + public ConsumerStatsManager getConsumerStatsManager() { + return this.mQClientFactory.getConsumerStatsManager(); + } + + + public Set queryConsumeTimeSpan(final String topic) throws RemotingException, + MQClientException, InterruptedException, MQBrokerException { + Set queueTimeSpan = new HashSet(); + TopicRouteData routeData = + this.mQClientFactory.getMQClientAPIImpl().getTopicRouteInfoFromNameServer(topic, 3000); + for (BrokerData brokerData : routeData.getBrokerDatas()) { + String addr = brokerData.selectBrokerAddr(); + queueTimeSpan.addAll(this.mQClientFactory.getMQClientAPIImpl().queryConsumeTimeSpan(addr, topic, + groupName(), 3000l)); + } + + return queueTimeSpan; + } + + + public ConsumeMessageService getConsumeMessageService() { + return consumeMessageService; + } + + + public void setConsumeMessageService(ConsumeMessageService consumeMessageService) { + this.consumeMessageService = consumeMessageService; + + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/MQConsumerInner.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/MQConsumerInner.java new file mode 100644 index 000000000..e4fa183bb --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/MQConsumerInner.java @@ -0,0 +1,66 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.impl.consumer; + +import java.util.Set; + +import com.alibaba.rocketmq.common.consumer.ConsumeFromWhere; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.common.protocol.body.ConsumerRunningInfo; +import com.alibaba.rocketmq.common.protocol.heartbeat.ConsumeType; +import com.alibaba.rocketmq.common.protocol.heartbeat.MessageModel; +import com.alibaba.rocketmq.common.protocol.heartbeat.SubscriptionData; + + +/** + * Consumer内部接口，供MQClientFactory使用 + * + * @author shijia.wxr + * @since 2013-7-24 + */ +public interface MQConsumerInner { + public String groupName(); + + + public MessageModel messageModel(); + + + public ConsumeType consumeType(); + + + public ConsumeFromWhere consumeFromWhere(); + + + public Set subscriptions(); + + + public void doRebalance(); + + + public void persistConsumerOffset(); + + + public void updateTopicSubscribeInfo(final String topic, final Set info); + + + public boolean isSubscribeTopicNeedUpdate(final String topic); + + + public boolean isUnitMode(); + + + public ConsumerRunningInfo consumerRunningInfo(); +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/MessageQueueLock.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/MessageQueueLock.java new file mode 100644 index 000000000..a03a27991 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/MessageQueueLock.java @@ -0,0 +1,46 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.impl.consumer; + +import java.util.concurrent.ConcurrentHashMap; + +import com.alibaba.rocketmq.common.message.MessageQueue; + + +/** + * 严格保证单个队列同一时刻只有一个线程消费 + * + * @author shijia.wxr + * @since 2013-6-25 + */ +public class MessageQueueLock { + private ConcurrentHashMap mqLockTable = + new ConcurrentHashMap(); + + + public Object fetchLockObject(final MessageQueue mq) { + Object objLock = this.mqLockTable.get(mq); + if (null == objLock) { + objLock = new Object(); + Object prevLock = this.mqLockTable.putIfAbsent(mq, objLock); + if (prevLock != null) { + objLock = prevLock; + } + } + + return objLock; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/ProcessQueue.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/ProcessQueue.java new file mode 100644 index 000000000..ab92c6fa7 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/ProcessQueue.java @@ -0,0 +1,452 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.impl.consumer; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +import org.slf4j.Logger; + +import com.alibaba.rocketmq.client.log.ClientLogger; +import com.alibaba.rocketmq.common.message.MessageConst; +import com.alibaba.rocketmq.common.message.MessageExt; +import com.alibaba.rocketmq.common.protocol.body.ProcessQueueInfo; + + +/** + * 正在被消费的队列，含消息 + * + * @author shijia.wxr + * @since 2013-7-24 + */ +public class ProcessQueue { + // 客户端本地Lock存活最大时间，超过则自动过期，单位ms + public final static long RebalanceLockMaxLiveTime = Long.parseLong(System.getProperty( + "rocketmq.client.rebalance.lockMaxLiveTime", "30000")); + // 定时Lock间隔时间，单位ms + public final static long RebalanceLockInterval = Long.parseLong(System.getProperty( + "rocketmq.client.rebalance.lockInterval", "20000")); + + private final Logger log = ClientLogger.getLog(); + private final ReadWriteLock lockTreeMap = new ReentrantReadWriteLock(); + private final TreeMap msgTreeMap = new TreeMap(); + private volatile long queueOffsetMax = 0L; + private final AtomicLong msgCount = new AtomicLong(); + + // 当前Q是否被rebalance丢弃 + private volatile boolean droped = false; + private volatile long lastPullTimestamp = System.currentTimeMillis(); + private final static long PullMaxIdleTime = Long.parseLong(System.getProperty( + "rocketmq.client.pull.pullMaxIdleTime", "120000")); + + // 最后一次消费的时间戳 + private volatile long lastConsumeTimestamp = System.currentTimeMillis(); + + /** + * 顺序消息专用 + */ + private final Lock lockConsume = new ReentrantLock(); + + // 是否从Broker锁定 + private volatile boolean locked = false; + // 最后一次锁定成功时间戳 + private volatile long lastLockTimestamp = System.currentTimeMillis(); + // 是否正在被消费 + private volatile boolean consuming = false; + // 事务方式消费，未提交的消息 + private final TreeMap msgTreeMapTemp = new TreeMap(); + // 尝试释放这个队列的次数 + private final AtomicLong tryUnlockTimes = new AtomicLong(0); + + /** + * 当前队列的消息堆积数量 + */ + private volatile long msgDuijiCnt = 0; + + + public boolean isLockExpired() { + boolean result = (System.currentTimeMillis() - this.lastLockTimestamp) > RebalanceLockMaxLiveTime; + return result; + } + + + public boolean isPullExpired() { + boolean result = (System.currentTimeMillis() - this.lastPullTimestamp) > PullMaxIdleTime; + return result; + } + + + /** + * @return 是否需要分发当前队列到消费线程池 + */ + public boolean putMessage(final List msgs) { + boolean dispathToConsume = false; + try { + this.lockTreeMap.writeLock().lockInterruptibly(); + try { + int validMsgCnt = 0; + for (MessageExt msg : msgs) { + MessageExt old = msgTreeMap.put(msg.getQueueOffset(), msg); + if (null == old) { + validMsgCnt++; + this.queueOffsetMax = msg.getQueueOffset(); + } + } + msgCount.addAndGet(validMsgCnt); + + if (!msgTreeMap.isEmpty() && !this.consuming) { + dispathToConsume = true; + this.consuming = true; + } + + // 计算当前队列堆积的消息数量 + if (!msgs.isEmpty()) { + MessageExt messageExt = msgs.get(msgs.size() - 1); + String property = messageExt.getProperty(MessageConst.PROPERTY_MAX_OFFSET); + if (property != null) { + long duiji = Long.parseLong(property) - messageExt.getQueueOffset(); + if (duiji > 0) { + this.msgDuijiCnt = duiji; + } + } + } + } + finally { + this.lockTreeMap.writeLock().unlock(); + } + } + catch (InterruptedException e) { + log.error("putMessage exception", e); + } + + return dispathToConsume; + } + + + /** + * 获取当前队列的最大跨度 + */ + public long getMaxSpan() { + try { + this.lockTreeMap.readLock().lockInterruptibly(); + try { + if (!this.msgTreeMap.isEmpty()) { + return this.msgTreeMap.lastKey() - this.msgTreeMap.firstKey(); + } + } + finally { + this.lockTreeMap.readLock().unlock(); + } + } + catch (InterruptedException e) { + log.error("getMaxSpan exception", e); + } + + return 0; + } + + + /** + * 删除已经消费过的消息，返回最小Offset，这个Offset对应的消息未消费 + * + * @param msgs + * @return + */ + public long removeMessage(final List msgs) { + long result = -1; + final long now = System.currentTimeMillis(); + try { + this.lockTreeMap.writeLock().lockInterruptibly(); + this.lastConsumeTimestamp = now; + try { + if (!msgTreeMap.isEmpty()) { + result = this.queueOffsetMax + 1; + int removedCnt = 0; + for (MessageExt msg : msgs) { + MessageExt prev = msgTreeMap.remove(msg.getQueueOffset()); + if (prev != null) { + removedCnt--; + } + } + msgCount.addAndGet(removedCnt); + + if (!msgTreeMap.isEmpty()) { + result = msgTreeMap.firstKey(); + } + } + } + finally { + this.lockTreeMap.writeLock().unlock(); + } + } + catch (Throwable t) { + log.error("removeMessage exception", t); + } + + return result; + } + + + public TreeMap getMsgTreeMap() { + return msgTreeMap; + } + + + public AtomicLong getMsgCount() { + return msgCount; + } + + + public boolean isDroped() { + return droped; + } + + + public void setDroped(boolean droped) { + this.droped = droped; + } + + + /** + * ======================================================================== + * 以下部分为顺序消息专有操作 + */ + + public void setLocked(boolean locked) { + this.locked = locked; + } + + + public boolean isLocked() { + return locked; + } + + + public void rollback() { + try { + this.lockTreeMap.writeLock().lockInterruptibly(); + try { + this.msgTreeMap.putAll(this.msgTreeMapTemp); + this.msgTreeMapTemp.clear(); + } + finally { + this.lockTreeMap.writeLock().unlock(); + } + } + catch (InterruptedException e) { + log.error("rollback exception", e); + } + } + + + public long commit() { + try { + this.lockTreeMap.writeLock().lockInterruptibly(); + try { + Long offset = this.msgTreeMapTemp.lastKey(); + msgCount.addAndGet(this.msgTreeMapTemp.size() * (-1)); + this.msgTreeMapTemp.clear(); + if (offset != null) { + return offset + 1; + } + } + finally { + this.lockTreeMap.writeLock().unlock(); + } + } + catch (InterruptedException e) { + log.error("commit exception", e); + } + + return -1; + } + + + public void makeMessageToCosumeAgain(List msgs) { + try { + this.lockTreeMap.writeLock().lockInterruptibly(); + try { + // 临时Table删除 + // 正常Table增加 + for (MessageExt msg : msgs) { + this.msgTreeMapTemp.remove(msg.getQueueOffset()); + this.msgTreeMap.put(msg.getQueueOffset(), msg); + } + } + finally { + this.lockTreeMap.writeLock().unlock(); + } + } + catch (InterruptedException e) { + log.error("makeMessageToCosumeAgain exception", e); + } + } + + + /** + * 如果取不到消息，则将正在消费状态置为false + * + * @param batchSize + * @return + */ + public List takeMessags(final int batchSize) { + List result = new ArrayList(batchSize); + final long now = System.currentTimeMillis(); + try { + this.lockTreeMap.writeLock().lockInterruptibly(); + this.lastConsumeTimestamp = now; + try { + if (!this.msgTreeMap.isEmpty()) { + for (int i = 0; i < batchSize; i++) { + Map.Entry entry = this.msgTreeMap.pollFirstEntry(); + if (entry != null) { + result.add(entry.getValue()); + msgTreeMapTemp.put(entry.getKey(), entry.getValue()); + } + else { + break; + } + } + } + + if (result.isEmpty()) { + consuming = false; + } + } + finally { + this.lockTreeMap.writeLock().unlock(); + } + } + catch (InterruptedException e) { + log.error("takeMessags exception", e); + } + + return result; + } + + + public void clear() { + try { + this.lockTreeMap.writeLock().lockInterruptibly(); + try { + this.msgTreeMap.clear(); + this.msgTreeMapTemp.clear(); + this.msgCount.set(0); + this.queueOffsetMax = 0L; + } + finally { + this.lockTreeMap.writeLock().unlock(); + } + } + catch (InterruptedException e) { + log.error("rollback exception", e); + } + } + + + public long getLastLockTimestamp() { + return lastLockTimestamp; + } + + + public void setLastLockTimestamp(long lastLockTimestamp) { + this.lastLockTimestamp = lastLockTimestamp; + } + + + public Lock getLockConsume() { + return lockConsume; + } + + + public long getLastPullTimestamp() { + return lastPullTimestamp; + } + + + public void setLastPullTimestamp(long lastPullTimestamp) { + this.lastPullTimestamp = lastPullTimestamp; + } + + + public long getMsgDuijiCnt() { + return msgDuijiCnt; + } + + + public void setMsgDuijiCnt(long msgDuijiCnt) { + this.msgDuijiCnt = msgDuijiCnt; + } + + + public long getTryUnlockTimes() { + return this.tryUnlockTimes.get(); + } + + + public void incTryUnlockTimes() { + this.tryUnlockTimes.incrementAndGet(); + } + + + public void fillProcessQueueInfo(final ProcessQueueInfo info) { + try { + this.lockTreeMap.readLock().lockInterruptibly(); + + if (!this.msgTreeMap.isEmpty()) { + info.setCachedMsgMinOffset(this.msgTreeMap.firstKey()); + info.setCachedMsgMaxOffset(this.msgTreeMap.lastKey()); + info.setCachedMsgCount(this.msgTreeMap.size()); + } + + if (!this.msgTreeMapTemp.isEmpty()) { + info.setTransactionMsgMinOffset(this.msgTreeMapTemp.firstKey()); + info.setTransactionMsgMaxOffset(this.msgTreeMapTemp.lastKey()); + info.setTransactionMsgCount(this.msgTreeMapTemp.size()); + } + + info.setLocked(this.locked); + info.setTryUnlockTimes(this.tryUnlockTimes.get()); + info.setLastLockTimestamp(this.lastLockTimestamp); + + info.setDroped(this.droped); + info.setLastPullTimestamp(this.lastPullTimestamp); + info.setLastConsumeTimestamp(this.lastConsumeTimestamp); + } + catch (Exception e) { + } + finally { + this.lockTreeMap.readLock().unlock(); + } + } + + + public long getLastConsumeTimestamp() { + return lastConsumeTimestamp; + } + + + public void setLastConsumeTimestamp(long lastConsumeTimestamp) { + this.lastConsumeTimestamp = lastConsumeTimestamp; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/PullAPIWrapper.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/PullAPIWrapper.java new file mode 100644 index 000000000..4a81ee8ba --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/PullAPIWrapper.java @@ -0,0 +1,329 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.impl.consumer; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; + +import org.slf4j.Logger; + +import com.alibaba.rocketmq.client.VirtualEnvUtil; +import com.alibaba.rocketmq.client.consumer.PullCallback; +import com.alibaba.rocketmq.client.consumer.PullResult; +import com.alibaba.rocketmq.client.consumer.PullStatus; +import com.alibaba.rocketmq.client.exception.MQBrokerException; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.client.hook.FilterMessageContext; +import com.alibaba.rocketmq.client.hook.FilterMessageHook; +import com.alibaba.rocketmq.client.impl.CommunicationMode; +import com.alibaba.rocketmq.client.impl.FindBrokerResult; +import com.alibaba.rocketmq.client.impl.factory.MQClientInstance; +import com.alibaba.rocketmq.client.log.ClientLogger; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.message.MessageAccessor; +import com.alibaba.rocketmq.common.message.MessageConst; +import com.alibaba.rocketmq.common.message.MessageDecoder; +import com.alibaba.rocketmq.common.message.MessageExt; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.common.protocol.header.PullMessageRequestHeader; +import com.alibaba.rocketmq.common.protocol.heartbeat.SubscriptionData; +import com.alibaba.rocketmq.common.protocol.route.TopicRouteData; +import com.alibaba.rocketmq.common.sysflag.PullSysFlag; +import com.alibaba.rocketmq.remoting.exception.RemotingException; + + +/** + * 对Pull接口进行进一步的封装 + * + * @author shijia.wxr + * @since 2013-7-24 + */ +public class PullAPIWrapper { + private final Logger log = ClientLogger.getLog(); + private ConcurrentHashMap pullFromWhichNodeTable = + new ConcurrentHashMap(32); + + private final MQClientInstance mQClientFactory; + private final String consumerGroup; + private final boolean unitMode; + + private volatile boolean connectBrokerByUser = false; + private volatile long defaultBrokerId = MixAll.MASTER_ID; + + + public PullAPIWrapper(MQClientInstance mQClientFactory, String consumerGroup, boolean unitMode) { + this.mQClientFactory = mQClientFactory; + this.consumerGroup = consumerGroup; + this.unitMode = unitMode; + } + + + public void updatePullFromWhichNode(final MessageQueue mq, final long brokerId) { + AtomicLong suggest = this.pullFromWhichNodeTable.get(mq); + if (null == suggest) { + this.pullFromWhichNodeTable.put(mq, new AtomicLong(brokerId)); + } + else { + suggest.set(brokerId); + } + } + + private Random random = new Random(System.currentTimeMillis()); + + + public int randomNum() { + int value = random.nextInt(); + if (value < 0) { + value = Math.abs(value); + if (value < 0) + value = 0; + } + return value; + } + + + /** + * 随机找Filter Server + * + * @param brokerAddr + * @return + * @throws MQClientException + */ + private String computPullFromWhichFilterServer(final String topic, final String brokerAddr) + throws MQClientException { + ConcurrentHashMap topicRouteTable = this.mQClientFactory.getTopicRouteTable(); + if (topicRouteTable != null) { + TopicRouteData topicRouteData = topicRouteTable.get(topic); + List list = topicRouteData.getFilterServerTable().get(brokerAddr); + + if (list != null && !list.isEmpty()) { + return list.get(randomNum() % list.size()); + } + } + + throw new MQClientException("Find Filter Server Failed, Broker Addr: " + brokerAddr + " topic: " + + topic, null); + } + + + /** + * 对拉取结果进行处理，主要是消息反序列化 + * + * @param mq + * @param pullResult + * @param subscriptionData + * @return + */ + public PullResult processPullResult(final MessageQueue mq, final PullResult pullResult, + final SubscriptionData subscriptionData) { + final String projectGroupPrefix = this.mQClientFactory.getMQClientAPIImpl().getProjectGroupPrefix(); + PullResultExt pullResultExt = (PullResultExt) pullResult; + + this.updatePullFromWhichNode(mq, pullResultExt.getSuggestWhichBrokerId()); + if (PullStatus.FOUND == pullResult.getPullStatus()) { + ByteBuffer byteBuffer = ByteBuffer.wrap(pullResultExt.getMessageBinary()); + List msgList = MessageDecoder.decodes(byteBuffer); + + // 消息再次过滤 + List msgListFilterAgain = msgList; + if (!subscriptionData.getTagsSet().isEmpty() && !subscriptionData.isClassFilterMode()) { + msgListFilterAgain = new ArrayList(msgList.size()); + for (MessageExt msg : msgList) { + if (msg.getTags() != null) { + if (subscriptionData.getTagsSet().contains(msg.getTags())) { + msgListFilterAgain.add(msg); + } + } + } + } + + // 执行消息过滤的 FilterMessageHook + if (this.hasHook()) { + FilterMessageContext filterMessageContext = new FilterMessageContext(); + filterMessageContext.setUnitMode(unitMode); + filterMessageContext.setMsgList(msgListFilterAgain); + this.executeHook(filterMessageContext); + } + + // 清除虚拟运行环境相关的projectGroupPrefix + if (!UtilAll.isBlank(projectGroupPrefix)) { + subscriptionData.setTopic(VirtualEnvUtil.clearProjectGroup(subscriptionData.getTopic(), + projectGroupPrefix)); + mq.setTopic(VirtualEnvUtil.clearProjectGroup(mq.getTopic(), projectGroupPrefix)); + for (MessageExt msg : msgListFilterAgain) { + msg.setTopic(VirtualEnvUtil.clearProjectGroup(msg.getTopic(), projectGroupPrefix)); + // 消息中放入队列的最大最小Offset，方便应用来感知消息堆积程度 + + MessageAccessor.putProperty(msg, MessageConst.PROPERTY_MIN_OFFSET, + Long.toString(pullResult.getMinOffset())); + MessageAccessor.putProperty(msg, MessageConst.PROPERTY_MAX_OFFSET, + Long.toString(pullResult.getMaxOffset())); + } + } + else { + // 消息中放入队列的最大最小Offset，方便应用来感知消息堆积程度 + for (MessageExt msg : msgListFilterAgain) { + MessageAccessor.putProperty(msg, MessageConst.PROPERTY_MIN_OFFSET, + Long.toString(pullResult.getMinOffset())); + MessageAccessor.putProperty(msg, MessageConst.PROPERTY_MAX_OFFSET, + Long.toString(pullResult.getMaxOffset())); + } + } + + pullResultExt.setMsgFoundList(msgListFilterAgain); + } + + // 令GC释放内存 + pullResultExt.setMessageBinary(null); + + return pullResult; + } + + + /** + * 每个队列都应该有相应的变量来保存从哪个服务器拉 + */ + public long recalculatePullFromWhichNode(final MessageQueue mq) { + if (this.isConnectBrokerByUser()) { + return this.defaultBrokerId; + } + + AtomicLong suggest = this.pullFromWhichNodeTable.get(mq); + if (suggest != null) { + return suggest.get(); + } + + return MixAll.MASTER_ID; + } + + + public PullResult pullKernelImpl(// + final MessageQueue mq,// 1 + final String subExpression,// 2 + final long subVersion,// 3 + final long offset,// 4 + final int maxNums,// 5 + final int sysFlag,// 6 + final long commitOffset,// 7 + final long brokerSuspendMaxTimeMillis,// 8 + final long timeoutMillis,// 9 + final CommunicationMode communicationMode,// 10 + final PullCallback pullCallback// 11 + ) throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + FindBrokerResult findBrokerResult = + this.mQClientFactory.findBrokerAddressInSubscribe(mq.getBrokerName(), + this.recalculatePullFromWhichNode(mq), false); + if (null == findBrokerResult) { + // TODO 此处可能对Name Server压力过大，需要调优 + this.mQClientFactory.updateTopicRouteInfoFromNameServer(mq.getTopic()); + findBrokerResult = + this.mQClientFactory.findBrokerAddressInSubscribe(mq.getBrokerName(), + this.recalculatePullFromWhichNode(mq), false); + } + + if (findBrokerResult != null) { + int sysFlagInner = sysFlag; + + // Slave不允许实时提交消费进度，可以定时提交 + if (findBrokerResult.isSlave()) { + sysFlagInner = PullSysFlag.clearCommitOffsetFlag(sysFlagInner); + } + + PullMessageRequestHeader requestHeader = new PullMessageRequestHeader(); + requestHeader.setConsumerGroup(this.consumerGroup); + requestHeader.setTopic(mq.getTopic()); + requestHeader.setQueueId(mq.getQueueId()); + requestHeader.setQueueOffset(offset); + requestHeader.setMaxMsgNums(maxNums); + requestHeader.setSysFlag(sysFlagInner); + requestHeader.setCommitOffset(commitOffset); + requestHeader.setSuspendTimeoutMillis(brokerSuspendMaxTimeMillis); + requestHeader.setSubscription(subExpression); + requestHeader.setSubVersion(subVersion); + + String brokerAddr = findBrokerResult.getBrokerAddr(); + if (PullSysFlag.hasClassFilterFlag(sysFlagInner)) { + brokerAddr = computPullFromWhichFilterServer(mq.getTopic(), brokerAddr); + } + + PullResult pullResult = this.mQClientFactory.getMQClientAPIImpl().pullMessage(// + brokerAddr,// + requestHeader,// + timeoutMillis,// + communicationMode,// + pullCallback); + + return pullResult; + } + + throw new MQClientException("The broker[" + mq.getBrokerName() + "] not exist", null); + } + + /** + * 从服务端拉消息之后，会执行 FilterMessageHook + */ + private ArrayList filterMessageHookList = new ArrayList(); + + + public boolean hasHook() { + return !this.filterMessageHookList.isEmpty(); + } + + + public void registerFilterMessageHook(ArrayList filterMessageHookList) { + this.filterMessageHookList = filterMessageHookList; + } + + + public void executeHook(final FilterMessageContext context) { + if (!this.filterMessageHookList.isEmpty()) { + for (FilterMessageHook hook : this.filterMessageHookList) { + try { + hook.filterMessage(context); + } + catch (Throwable e) { + log.error("execute hook error. hookName={}", hook.hookName()); + } + } + } + } + + + public long getDefaultBrokerId() { + return defaultBrokerId; + } + + + public void setDefaultBrokerId(long defaultBrokerId) { + this.defaultBrokerId = defaultBrokerId; + } + + + public boolean isConnectBrokerByUser() { + return connectBrokerByUser; + } + + + public void setConnectBrokerByUser(boolean connectBrokerByUser) { + this.connectBrokerByUser = connectBrokerByUser; + + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/PullMessageService.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/PullMessageService.java new file mode 100644 index 000000000..6f87a539c --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/PullMessageService.java @@ -0,0 +1,133 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.impl.consumer; + +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; + +import org.slf4j.Logger; + +import com.alibaba.rocketmq.client.impl.factory.MQClientInstance; +import com.alibaba.rocketmq.client.log.ClientLogger; +import com.alibaba.rocketmq.common.ServiceThread; + + +/** + * 长轮询拉消息服务，单线程异步拉取 + * + * @author shijia.wxr + * @since 2013-7-24 + */ +public class PullMessageService extends ServiceThread { + private final Logger log = ClientLogger.getLog(); + private final LinkedBlockingQueue pullRequestQueue = new LinkedBlockingQueue(); + private final MQClientInstance mQClientFactory; + private final ScheduledExecutorService scheduledExecutorService = Executors + .newSingleThreadScheduledExecutor(new ThreadFactory() { + @Override + public Thread newThread(Runnable r) { + return new Thread(r, "PullMessageServiceScheduledThread"); + } + });; + + + public PullMessageService(MQClientInstance mQClientFactory) { + this.mQClientFactory = mQClientFactory; + } + + + /** + * 只定时一次 + */ + public void executePullRequestLater(final PullRequest pullRequest, final long timeDelay) { + this.scheduledExecutorService.schedule(new Runnable() { + + @Override + public void run() { + PullMessageService.this.executePullRequestImmediately(pullRequest); + } + }, timeDelay, TimeUnit.MILLISECONDS); + } + + + /** + * 只定时一次 + */ + public void executeTaskLater(final Runnable r, final long timeDelay) { + this.scheduledExecutorService.schedule(r, timeDelay, TimeUnit.MILLISECONDS); + } + + + /** + * 立刻执行PullRequest + */ + public void executePullRequestImmediately(final PullRequest pullRequest) { + try { + this.pullRequestQueue.put(pullRequest); + } + catch (InterruptedException e) { + log.error("executePullRequestImmediately pullRequestQueue.put", e); + } + } + + + private void pullMessage(final PullRequest pullRequest) { + final MQConsumerInner consumer = this.mQClientFactory.selectConsumer(pullRequest.getConsumerGroup()); + if (consumer != null) { + DefaultMQPushConsumerImpl impl = (DefaultMQPushConsumerImpl) consumer; + impl.pullMessage(pullRequest); + } + else { + log.warn("No matched consumer for the PullRequest {}, drop it", pullRequest); + } + } + + + @Override + public void run() { + log.info(this.getServiceName() + " service started"); + + while (!this.isStoped()) { + try { + PullRequest pullRequest = this.pullRequestQueue.take(); + if (pullRequest != null) { + this.pullMessage(pullRequest); + } + } + catch (InterruptedException e) { + } + catch (Exception e) { + log.error("Pull Message Service Run Method exception", e); + } + } + + log.info(this.getServiceName() + " service end"); + } + + + @Override + public String getServiceName() { + return PullMessageService.class.getSimpleName(); + } + + + public ScheduledExecutorService getScheduledExecutorService() { + return scheduledExecutorService; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/PullRequest.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/PullRequest.java new file mode 100644 index 000000000..2796c6650 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/PullRequest.java @@ -0,0 +1,115 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.impl.consumer; + +import com.alibaba.rocketmq.common.message.MessageQueue; + + +/** + * 拉消息请求 + * + * @author shijia.wxr + * @since 2013-7-24 + */ +public class PullRequest { + private String consumerGroup; + private MessageQueue messageQueue; + private ProcessQueue processQueue; + // hashCode与equals方法不包含此字段 + private long nextOffset; + + + public String getConsumerGroup() { + return consumerGroup; + } + + + public void setConsumerGroup(String consumerGroup) { + this.consumerGroup = consumerGroup; + } + + + public MessageQueue getMessageQueue() { + return messageQueue; + } + + + public void setMessageQueue(MessageQueue messageQueue) { + this.messageQueue = messageQueue; + } + + + public long getNextOffset() { + return nextOffset; + } + + + public void setNextOffset(long nextOffset) { + this.nextOffset = nextOffset; + } + + + @Override + public String toString() { + return "PullRequest [consumerGroup=" + consumerGroup + ", messageQueue=" + messageQueue + + ", nextOffset=" + nextOffset + "]"; + } + + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((consumerGroup == null) ? 0 : consumerGroup.hashCode()); + result = prime * result + ((messageQueue == null) ? 0 : messageQueue.hashCode()); + return result; + } + + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + PullRequest other = (PullRequest) obj; + if (consumerGroup == null) { + if (other.consumerGroup != null) + return false; + } + else if (!consumerGroup.equals(other.consumerGroup)) + return false; + if (messageQueue == null) { + if (other.messageQueue != null) + return false; + } + else if (!messageQueue.equals(other.messageQueue)) + return false; + return true; + } + + + public ProcessQueue getProcessQueue() { + return processQueue; + } + + + public void setProcessQueue(ProcessQueue processQueue) { + this.processQueue = processQueue; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/PullResultExt.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/PullResultExt.java new file mode 100644 index 000000000..a8bc35c1e --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/PullResultExt.java @@ -0,0 +1,57 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.impl.consumer; + +import java.util.List; + +import com.alibaba.rocketmq.client.consumer.PullResult; +import com.alibaba.rocketmq.client.consumer.PullStatus; +import com.alibaba.rocketmq.common.message.MessageExt; + + +/** + * 只在内部使用，不对外公开 + * + * @author shijia.wxr + * @since 2013-7-24 + */ +public class PullResultExt extends PullResult { + private final long suggestWhichBrokerId; + private byte[] messageBinary; + + + public PullResultExt(PullStatus pullStatus, long nextBeginOffset, long minOffset, long maxOffset, + List msgFoundList, final long suggestWhichBrokerId, final byte[] messageBinary) { + super(pullStatus, nextBeginOffset, minOffset, maxOffset, msgFoundList); + this.suggestWhichBrokerId = suggestWhichBrokerId; + this.messageBinary = messageBinary; + } + + + public byte[] getMessageBinary() { + return messageBinary; + } + + + public void setMessageBinary(byte[] messageBinary) { + this.messageBinary = messageBinary; + } + + + public long getSuggestWhichBrokerId() { + return suggestWhichBrokerId; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/RebalanceImpl.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/RebalanceImpl.java new file mode 100644 index 000000000..1abf8b02c --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/RebalanceImpl.java @@ -0,0 +1,533 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.impl.consumer; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +import org.slf4j.Logger; + +import com.alibaba.rocketmq.client.consumer.AllocateMessageQueueStrategy; +import com.alibaba.rocketmq.client.impl.FindBrokerResult; +import com.alibaba.rocketmq.client.impl.factory.MQClientInstance; +import com.alibaba.rocketmq.client.log.ClientLogger; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.common.protocol.body.LockBatchRequestBody; +import com.alibaba.rocketmq.common.protocol.body.UnlockBatchRequestBody; +import com.alibaba.rocketmq.common.protocol.heartbeat.ConsumeType; +import com.alibaba.rocketmq.common.protocol.heartbeat.MessageModel; +import com.alibaba.rocketmq.common.protocol.heartbeat.SubscriptionData; + + +/** + * Rebalance的具体实现 + * + * @author shijia.wxr + * @since 2013-6-22 + */ +public abstract class RebalanceImpl { + protected static final Logger log = ClientLogger.getLog(); + // 分配好的队列，消息存储也在这里 + protected final ConcurrentHashMap processQueueTable = + new ConcurrentHashMap(64); + // 可以订阅的所有队列（定时从Name Server更新最新版本） + protected final ConcurrentHashMap> topicSubscribeInfoTable = + new ConcurrentHashMap>(); + // 订阅关系，用户配置的原始数据 + protected final ConcurrentHashMap subscriptionInner = + new ConcurrentHashMap(); + protected String consumerGroup; + protected MessageModel messageModel; + protected AllocateMessageQueueStrategy allocateMessageQueueStrategy; + protected MQClientInstance mQClientFactory; + + + public RebalanceImpl(String consumerGroup, MessageModel messageModel, + AllocateMessageQueueStrategy allocateMessageQueueStrategy, MQClientInstance mQClientFactory) { + this.consumerGroup = consumerGroup; + this.messageModel = messageModel; + this.allocateMessageQueueStrategy = allocateMessageQueueStrategy; + this.mQClientFactory = mQClientFactory; + } + + + public abstract ConsumeType consumeType(); + + + public void unlock(final MessageQueue mq, final boolean oneway) { + FindBrokerResult findBrokerResult = + this.mQClientFactory.findBrokerAddressInSubscribe(mq.getBrokerName(), MixAll.MASTER_ID, true); + if (findBrokerResult != null) { + UnlockBatchRequestBody requestBody = new UnlockBatchRequestBody(); + requestBody.setConsumerGroup(this.consumerGroup); + requestBody.setClientId(this.mQClientFactory.getClientId()); + requestBody.getMqSet().add(mq); + + try { + this.mQClientFactory.getMQClientAPIImpl().unlockBatchMQ(findBrokerResult.getBrokerAddr(), + requestBody, 1000, oneway); + log.warn("unlock messageQueue. group:{}, clientId:{}, mq:{}",// + this.consumerGroup, // + this.mQClientFactory.getClientId(), // + mq); + } + catch (Exception e) { + log.error("unlockBatchMQ exception, " + mq, e); + } + } + } + + + public void unlockAll(final boolean oneway) { + HashMap> brokerMqs = this.buildProcessQueueTableByBrokerName(); + + for (final Map.Entry> entry : brokerMqs.entrySet()) { + final String brokerName = entry.getKey(); + final Set mqs = entry.getValue(); + + if (mqs.isEmpty()) + continue; + + FindBrokerResult findBrokerResult = + this.mQClientFactory.findBrokerAddressInSubscribe(brokerName, MixAll.MASTER_ID, true); + if (findBrokerResult != null) { + UnlockBatchRequestBody requestBody = new UnlockBatchRequestBody(); + requestBody.setConsumerGroup(this.consumerGroup); + requestBody.setClientId(this.mQClientFactory.getClientId()); + requestBody.setMqSet(mqs); + + try { + this.mQClientFactory.getMQClientAPIImpl().unlockBatchMQ(findBrokerResult.getBrokerAddr(), + requestBody, 1000, oneway); + + for (MessageQueue mq : mqs) { + ProcessQueue processQueue = this.processQueueTable.get(mq); + if (processQueue != null) { + processQueue.setLocked(false); + log.info("the message queue unlock OK, Group: {} {}", this.consumerGroup, mq); + } + } + } + catch (Exception e) { + log.error("unlockBatchMQ exception, " + mqs, e); + } + } + } + } + + + private HashMap> buildProcessQueueTableByBrokerName() { + HashMap> result = new HashMap>(); + for (MessageQueue mq : this.processQueueTable.keySet()) { + Set mqs = result.get(mq.getBrokerName()); + if (null == mqs) { + mqs = new HashSet(); + result.put(mq.getBrokerName(), mqs); + } + + mqs.add(mq); + } + + return result; + } + + + public boolean lock(final MessageQueue mq) { + FindBrokerResult findBrokerResult = + this.mQClientFactory.findBrokerAddressInSubscribe(mq.getBrokerName(), MixAll.MASTER_ID, true); + if (findBrokerResult != null) { + LockBatchRequestBody requestBody = new LockBatchRequestBody(); + requestBody.setConsumerGroup(this.consumerGroup); + requestBody.setClientId(this.mQClientFactory.getClientId()); + requestBody.getMqSet().add(mq); + + try { + Set lockedMq = + this.mQClientFactory.getMQClientAPIImpl().lockBatchMQ( + findBrokerResult.getBrokerAddr(), requestBody, 1000); + for (MessageQueue mmqq : lockedMq) { + ProcessQueue processQueue = this.processQueueTable.get(mmqq); + if (processQueue != null) { + processQueue.setLocked(true); + processQueue.setLastLockTimestamp(System.currentTimeMillis()); + } + } + + boolean lockOK = lockedMq.contains(mq); + log.info("the message queue lock {}, {} {}",// + (lockOK ? "OK" : "Failed"), // + this.consumerGroup, // + mq); + return lockOK; + } + catch (Exception e) { + log.error("lockBatchMQ exception, " + mq, e); + } + } + + return false; + } + + + public void lockAll() { + HashMap> brokerMqs = this.buildProcessQueueTableByBrokerName(); + + Iterator>> it = brokerMqs.entrySet().iterator(); + while (it.hasNext()) { + Entry> entry = it.next(); + final String brokerName = entry.getKey(); + final Set mqs = entry.getValue(); + + if (mqs.isEmpty()) + continue; + + FindBrokerResult findBrokerResult = + this.mQClientFactory.findBrokerAddressInSubscribe(brokerName, MixAll.MASTER_ID, true); + if (findBrokerResult != null) { + LockBatchRequestBody requestBody = new LockBatchRequestBody(); + requestBody.setConsumerGroup(this.consumerGroup); + requestBody.setClientId(this.mQClientFactory.getClientId()); + requestBody.setMqSet(mqs); + + try { + Set lockOKMQSet = + this.mQClientFactory.getMQClientAPIImpl().lockBatchMQ( + findBrokerResult.getBrokerAddr(), requestBody, 1000); + + // 锁定成功的队列 + for (MessageQueue mq : lockOKMQSet) { + ProcessQueue processQueue = this.processQueueTable.get(mq); + if (processQueue != null) { + if (!processQueue.isLocked()) { + log.info("the message queue locked OK, Group: {} {}", this.consumerGroup, mq); + } + + processQueue.setLocked(true); + processQueue.setLastLockTimestamp(System.currentTimeMillis()); + } + } + // 锁定失败的队列 + for (MessageQueue mq : mqs) { + if (!lockOKMQSet.contains(mq)) { + ProcessQueue processQueue = this.processQueueTable.get(mq); + if (processQueue != null) { + processQueue.setLocked(false); + log.warn("the message queue locked Failed, Group: {} {}", this.consumerGroup, + mq); + } + } + } + } + catch (Exception e) { + log.error("lockBatchMQ exception, " + mqs, e); + } + } + } + } + + + public void doRebalance() { + Map subTable = this.getSubscriptionInner(); + if (subTable != null) { + for (final Map.Entry entry : subTable.entrySet()) { + final String topic = entry.getKey(); + try { + this.rebalanceByTopic(topic); + } + catch (Exception e) { + if (!topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) { + log.warn("rebalanceByTopic Exception", e); + } + } + } + } + + this.truncateMessageQueueNotMyTopic(); + } + + + private void rebalanceByTopic(final String topic) { + switch (messageModel) { + case BROADCASTING: { + Set mqSet = this.topicSubscribeInfoTable.get(topic); + if (mqSet != null) { + boolean changed = this.updateProcessQueueTableInRebalance(topic, mqSet); + if (changed) { + this.messageQueueChanged(topic, mqSet, mqSet); + log.info("messageQueueChanged {} {} {} {}",// + consumerGroup,// + topic,// + mqSet,// + mqSet); + } + } + else { + log.warn("doRebalance, {}, but the topic[{}] not exist.", consumerGroup, topic); + } + break; + } + case CLUSTERING: { + Set mqSet = this.topicSubscribeInfoTable.get(topic); + List cidAll = this.mQClientFactory.findConsumerIdList(topic, consumerGroup); + if (null == mqSet) { + if (!topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) { + log.warn("doRebalance, {}, but the topic[{}] not exist.", consumerGroup, topic); + } + } + + if (null == cidAll) { + log.warn("doRebalance, {} {}, get consumer id list failed", consumerGroup, topic); + } + + if (mqSet != null && cidAll != null) { + List mqAll = new ArrayList(); + mqAll.addAll(mqSet); + + // 排序 + Collections.sort(mqAll); + Collections.sort(cidAll); + + AllocateMessageQueueStrategy strategy = this.allocateMessageQueueStrategy; + + // 执行分配算法 + List allocateResult = null; + try { + allocateResult = strategy.allocate(// + this.consumerGroup, // + this.mQClientFactory.getClientId(), // + mqAll,// + cidAll); + } + catch (Throwable e) { + log.error( + "AllocateMessageQueueStrategy.allocate Exception. allocateMessageQueueStrategyName={}", + strategy.getName(), e); + return; + } + + Set allocateResultSet = new HashSet(); + if (allocateResult != null) { + allocateResultSet.addAll(allocateResult); + } + + // 更新本地队列 + boolean changed = this.updateProcessQueueTableInRebalance(topic, allocateResultSet); + if (changed) { + log.info( + "rebalanced allocate source. allocateMessageQueueStrategyName={}, group={}, topic={}, mqAllSize={}, cidAllSize={}, mqAll={}, cidAll={}", + strategy.getName(), consumerGroup, topic, mqSet.size(), cidAll.size(), mqSet, cidAll); + log.info( + "rebalanced result changed. allocateMessageQueueStrategyName={}, group={}, topic={}, ConsumerId={}, rebalanceSize={}, rebalanceMqSet={}", + strategy.getName(), consumerGroup, topic, this.mQClientFactory.getClientId(), + allocateResultSet.size(), mqAll.size(), cidAll.size(), allocateResultSet); + + this.messageQueueChanged(topic, mqSet, allocateResultSet); + } + } + break; + } + default: + break; + } + } + + + public abstract void messageQueueChanged(final String topic, final Set mqAll, + final Set mqDivided); + + + public void removeProcessQueue(final MessageQueue mq) { + ProcessQueue prev = this.processQueueTable.remove(mq); + if (prev != null) { + boolean droped = prev.isDroped(); + prev.setDroped(true); + this.removeUnnecessaryMessageQueue(mq, prev); + log.info("Fix Offset, {}, remove unnecessary mq, {} Droped: {}", consumerGroup, mq, droped); + } + } + + + private boolean updateProcessQueueTableInRebalance(final String topic, final Set mqSet) { + boolean changed = false; + + // 将多余的队列删除 + Iterator> it = this.processQueueTable.entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + MessageQueue mq = next.getKey(); + ProcessQueue pq = next.getValue(); + + if (mq.getTopic().equals(topic)) { + if (!mqSet.contains(mq)) { + pq.setDroped(true); + if (this.removeUnnecessaryMessageQueue(mq, pq)) { + it.remove(); + changed = true; + log.info("doRebalance, {}, remove unnecessary mq, {}", consumerGroup, mq); + } + } + // 超过2分钟没有拉取动作了，删除它 + else if (pq.isPullExpired()) { + switch (this.consumeType()) { + case CONSUME_ACTIVELY: + break; + case CONSUME_PASSIVELY: + pq.setDroped(true); + if (this.removeUnnecessaryMessageQueue(mq, pq)) { + it.remove(); + changed = true; + log.error( + "[BUG]doRebalance, {}, remove unnecessary mq, {}, because pull is pause, so try to fixed it", + consumerGroup, mq); + } + break; + default: + break; + } + } + } + } + + // 增加新增的队列 + List pullRequestList = new ArrayList(); + for (MessageQueue mq : mqSet) { + if (!this.processQueueTable.containsKey(mq)) { + PullRequest pullRequest = new PullRequest(); + pullRequest.setConsumerGroup(consumerGroup); + pullRequest.setMessageQueue(mq); + pullRequest.setProcessQueue(new ProcessQueue()); + + // 这个需要根据策略来设置 + long nextOffset = this.computePullFromWhere(mq); + if (nextOffset >= 0) { + pullRequest.setNextOffset(nextOffset); + pullRequestList.add(pullRequest); + changed = true; + this.processQueueTable.put(mq, pullRequest.getProcessQueue()); + log.info("doRebalance, {}, add a new mq, {}", consumerGroup, mq); + } + else { + // 等待此次Rebalance做重试 + log.warn("doRebalance, {}, add new mq failed, {}", consumerGroup, mq); + } + } + } + + this.dispatchPullRequest(pullRequestList); + + return changed; + } + + + public abstract boolean removeUnnecessaryMessageQueue(final MessageQueue mq, final ProcessQueue pq); + + + public abstract void dispatchPullRequest(final List pullRequestList); + + + public abstract long computePullFromWhere(final MessageQueue mq); + + + private void truncateMessageQueueNotMyTopic() { + Map subTable = this.getSubscriptionInner(); + + for (MessageQueue mq : this.processQueueTable.keySet()) { + if (!subTable.containsKey(mq.getTopic())) { + ProcessQueue pq = this.processQueueTable.remove(mq); + if (pq != null) { + pq.setDroped(true); + log.info("doRebalance, {}, truncateMessageQueueNotMyTopic remove unnecessary mq, {}", + consumerGroup, mq); + } + } + } + } + + + public ConcurrentHashMap getSubscriptionInner() { + return subscriptionInner; + } + + + public ConcurrentHashMap getProcessQueueTable() { + return processQueueTable; + } + + + public ConcurrentHashMap> getTopicSubscribeInfoTable() { + return topicSubscribeInfoTable; + } + + + public String getConsumerGroup() { + return consumerGroup; + } + + + public void setConsumerGroup(String consumerGroup) { + this.consumerGroup = consumerGroup; + } + + + public MessageModel getMessageModel() { + return messageModel; + } + + + public void setMessageModel(MessageModel messageModel) { + this.messageModel = messageModel; + } + + + public AllocateMessageQueueStrategy getAllocateMessageQueueStrategy() { + return allocateMessageQueueStrategy; + } + + + public void setAllocateMessageQueueStrategy(AllocateMessageQueueStrategy allocateMessageQueueStrategy) { + this.allocateMessageQueueStrategy = allocateMessageQueueStrategy; + } + + + public MQClientInstance getmQClientFactory() { + return mQClientFactory; + } + + + public void setmQClientFactory(MQClientInstance mQClientFactory) { + this.mQClientFactory = mQClientFactory; + } + + + public void destroy() { + Iterator> it = this.processQueueTable.entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + next.getValue().setDroped(true); + } + + this.processQueueTable.clear(); + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/RebalancePullImpl.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/RebalancePullImpl.java new file mode 100644 index 000000000..c783d5e15 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/RebalancePullImpl.java @@ -0,0 +1,88 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.impl.consumer; + +import java.util.List; +import java.util.Set; + +import com.alibaba.rocketmq.client.consumer.AllocateMessageQueueStrategy; +import com.alibaba.rocketmq.client.consumer.MessageQueueListener; +import com.alibaba.rocketmq.client.impl.factory.MQClientInstance; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.common.protocol.heartbeat.ConsumeType; +import com.alibaba.rocketmq.common.protocol.heartbeat.MessageModel; + + +/** + * @author shijia.wxr + * @since 2013-6-22 + */ +public class RebalancePullImpl extends RebalanceImpl { + private final DefaultMQPullConsumerImpl defaultMQPullConsumerImpl; + + + public RebalancePullImpl(DefaultMQPullConsumerImpl defaultMQPullConsumerImpl) { + this(null, null, null, null, defaultMQPullConsumerImpl); + } + + + public RebalancePullImpl(String consumerGroup, MessageModel messageModel, + AllocateMessageQueueStrategy allocateMessageQueueStrategy, MQClientInstance mQClientFactory, + DefaultMQPullConsumerImpl defaultMQPullConsumerImpl) { + super(consumerGroup, messageModel, allocateMessageQueueStrategy, mQClientFactory); + this.defaultMQPullConsumerImpl = defaultMQPullConsumerImpl; + } + + + @Override + public long computePullFromWhere(MessageQueue mq) { + return 0; + } + + + @Override + public void dispatchPullRequest(List pullRequestList) { + } + + + @Override + public void messageQueueChanged(String topic, Set mqAll, Set mqDivided) { + MessageQueueListener messageQueueListener = + this.defaultMQPullConsumerImpl.getDefaultMQPullConsumer().getMessageQueueListener(); + if (messageQueueListener != null) { + try { + messageQueueListener.messageQueueChanged(topic, mqAll, mqDivided); + } + catch (Throwable e) { + log.error("messageQueueChanged exception", e); + } + } + } + + + @Override + public boolean removeUnnecessaryMessageQueue(MessageQueue mq, ProcessQueue pq) { + this.defaultMQPullConsumerImpl.getOffsetStore().persist(mq); + this.defaultMQPullConsumerImpl.getOffsetStore().removeOffset(mq); + return true; + } + + + @Override + public ConsumeType consumeType() { + return ConsumeType.CONSUME_ACTIVELY; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/RebalancePushImpl.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/RebalancePushImpl.java new file mode 100644 index 000000000..afcd2885f --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/RebalancePushImpl.java @@ -0,0 +1,212 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.impl.consumer; + +import java.util.List; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +import com.alibaba.rocketmq.client.consumer.AllocateMessageQueueStrategy; +import com.alibaba.rocketmq.client.consumer.store.OffsetStore; +import com.alibaba.rocketmq.client.consumer.store.ReadOffsetType; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.client.impl.factory.MQClientInstance; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.consumer.ConsumeFromWhere; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.common.protocol.heartbeat.ConsumeType; +import com.alibaba.rocketmq.common.protocol.heartbeat.MessageModel; + + +/** + * @author shijia.wxr + * @since 2013-6-22 + */ +public class RebalancePushImpl extends RebalanceImpl { + private final DefaultMQPushConsumerImpl defaultMQPushConsumerImpl; + + + public RebalancePushImpl(DefaultMQPushConsumerImpl defaultMQPushConsumerImpl) { + this(null, null, null, null, defaultMQPushConsumerImpl); + } + + + public RebalancePushImpl(String consumerGroup, MessageModel messageModel, + AllocateMessageQueueStrategy allocateMessageQueueStrategy, MQClientInstance mQClientFactory, + DefaultMQPushConsumerImpl defaultMQPushConsumerImpl) { + super(consumerGroup, messageModel, allocateMessageQueueStrategy, mQClientFactory); + this.defaultMQPushConsumerImpl = defaultMQPushConsumerImpl; + } + + + @Override + public void dispatchPullRequest(List pullRequestList) { + // 派发PullRequest + for (PullRequest pullRequest : pullRequestList) { + this.defaultMQPushConsumerImpl.executePullRequestImmediately(pullRequest); + log.info("doRebalance, {}, add a new pull request {}", consumerGroup, pullRequest); + } + } + + + @Override + public long computePullFromWhere(MessageQueue mq) { + // 如果返回-1，这个队列的rebalance会失败重试，但是不影响其他队列。 + long result = -1; + final ConsumeFromWhere consumeFromWhere = + this.defaultMQPushConsumerImpl.getDefaultMQPushConsumer().getConsumeFromWhere(); + final OffsetStore offsetStore = this.defaultMQPushConsumerImpl.getOffsetStore(); + switch (consumeFromWhere) { + case CONSUME_FROM_LAST_OFFSET_AND_FROM_MIN_WHEN_BOOT_FIRST: + case CONSUME_FROM_MIN_OFFSET: + case CONSUME_FROM_MAX_OFFSET: + case CONSUME_FROM_LAST_OFFSET: { + long lastOffset = offsetStore.readOffset(mq, ReadOffsetType.READ_FROM_STORE); + // 第二次启动，根据上次的消费位点开始消费 + if (lastOffset >= 0) { + result = lastOffset; + } + // 第一次启动，没有记录消费位点 + else if (-1 == lastOffset) { + // 重试队列则从队列头部开始 + if (mq.getTopic().startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) { + result = 0L; + } + // 正常队列则从队列尾部开始 + else { + try { + result = this.mQClientFactory.getMQAdminImpl().maxOffset(mq); + } + catch (MQClientException e) { + result = -1; + } + } + } + // 其他错误 + else { + result = -1; + } + break; + } + case CONSUME_FROM_FIRST_OFFSET: { + long lastOffset = offsetStore.readOffset(mq, ReadOffsetType.READ_FROM_STORE); + // 第二次启动，根据上次的消费位点开始消费 + if (lastOffset >= 0) { + result = lastOffset; + } + // 第一次启动，没有记录消费位点 + else if (-1 == lastOffset) { + result = 0L; + } + // 其他错误 + else { + result = -1; + } + break; + } + case CONSUME_FROM_TIMESTAMP: { + long lastOffset = offsetStore.readOffset(mq, ReadOffsetType.READ_FROM_STORE); + // 第二次启动，根据上次的消费位点开始消费 + if (lastOffset >= 0) { + result = lastOffset; + } + // 第一次启动，没有记录消费为点 + else if (-1 == lastOffset) { + // 重试队列则从队列尾部开始 + if (mq.getTopic().startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) { + try { + result = this.mQClientFactory.getMQAdminImpl().maxOffset(mq); + } + catch (MQClientException e) { + result = -1; + } + } + // 正常队列则从指定时间点开始 + else { + try { + // 时间点需要参数配置 + long timestamp = + UtilAll.parseDate( + this.defaultMQPushConsumerImpl.getDefaultMQPushConsumer() + .getConsumeTimestamp(), UtilAll.yyyyMMddHHmmss).getTime(); + result = this.mQClientFactory.getMQAdminImpl().searchOffset(mq, timestamp); + } + catch (MQClientException e) { + result = -1; + } + } + } + // 其他错误 + else { + result = -1; + } + break; + } + + default: + break; + } + + return result; + } + + + @Override + public void messageQueueChanged(String topic, Set mqAll, Set mqDivided) { + } + + + @Override + public boolean removeUnnecessaryMessageQueue(MessageQueue mq, ProcessQueue pq) { + this.defaultMQPushConsumerImpl.getOffsetStore().persist(mq); + this.defaultMQPushConsumerImpl.getOffsetStore().removeOffset(mq); + if (this.defaultMQPushConsumerImpl.isConsumeOrderly() + && MessageModel.CLUSTERING.equals(this.defaultMQPushConsumerImpl.messageModel())) { + try { + if (pq.getLockConsume().tryLock(1000, TimeUnit.MILLISECONDS)) { + try { + this.unlock(mq, true); + return true; + } + finally { + pq.getLockConsume().unlock(); + } + } + else { + log.warn( + "[WRONG]mq is consuming, so can not unlock it, {}. maybe hanged for a while, {}",// + mq,// + pq.getTryUnlockTimes()); + + pq.incTryUnlockTimes(); + } + } + catch (Exception e) { + log.error("removeUnnecessaryMessageQueue Exception", e); + } + + return false; + } + return true; + } + + + @Override + public ConsumeType consumeType() { + return ConsumeType.CONSUME_PASSIVELY; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/RebalanceService.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/RebalanceService.java new file mode 100644 index 000000000..6561bddab --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/consumer/RebalanceService.java @@ -0,0 +1,60 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.impl.consumer; + +import org.slf4j.Logger; + +import com.alibaba.rocketmq.client.impl.factory.MQClientInstance; +import com.alibaba.rocketmq.client.log.ClientLogger; +import com.alibaba.rocketmq.common.ServiceThread; + + +/** + * Rebalance服务 + * + * @author shijia.wxr + * @since 2013-7-24 + */ +public class RebalanceService extends ServiceThread { + private final Logger log = ClientLogger.getLog(); + private final MQClientInstance mqClientFactory; + + + public RebalanceService(MQClientInstance mqClientFactory) { + this.mqClientFactory = mqClientFactory; + } + + private static long WaitInterval = 1000 * 10; + + + @Override + public void run() { + log.info(this.getServiceName() + " service started"); + + while (!this.isStoped()) { + this.waitForRunning(WaitInterval); + this.mqClientFactory.doRebalance(); + } + + log.info(this.getServiceName() + " service end"); + } + + + @Override + public String getServiceName() { + return RebalanceService.class.getSimpleName(); + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/factory/MQClientInstance.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/factory/MQClientInstance.java new file mode 100644 index 000000000..7140a4a8b --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/factory/MQClientInstance.java @@ -0,0 +1,1300 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.impl.factory; + +import com.alibaba.rocketmq.client.ClientConfig; +import com.alibaba.rocketmq.client.admin.MQAdminExtInner; +import com.alibaba.rocketmq.client.exception.MQBrokerException; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.client.impl.*; +import com.alibaba.rocketmq.client.impl.consumer.*; +import com.alibaba.rocketmq.client.impl.producer.DefaultMQProducerImpl; +import com.alibaba.rocketmq.client.impl.producer.MQProducerInner; +import com.alibaba.rocketmq.client.impl.producer.TopicPublishInfo; +import com.alibaba.rocketmq.client.log.ClientLogger; +import com.alibaba.rocketmq.client.producer.DefaultMQProducer; +import com.alibaba.rocketmq.client.stat.ConsumerStatsManager; +import com.alibaba.rocketmq.common.MQVersion; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.ServiceState; +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.constant.PermName; +import com.alibaba.rocketmq.common.filter.FilterAPI; +import com.alibaba.rocketmq.common.message.MessageExt; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.common.protocol.body.ConsumeMessageDirectlyResult; +import com.alibaba.rocketmq.common.protocol.body.ConsumerRunningInfo; +import com.alibaba.rocketmq.common.protocol.heartbeat.*; +import com.alibaba.rocketmq.common.protocol.route.BrokerData; +import com.alibaba.rocketmq.common.protocol.route.QueueData; +import com.alibaba.rocketmq.common.protocol.route.TopicRouteData; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.remoting.common.RemotingHelper; +import com.alibaba.rocketmq.remoting.exception.RemotingException; +import com.alibaba.rocketmq.remoting.netty.NettyClientConfig; +import org.slf4j.Logger; + +import java.io.UnsupportedEncodingException; +import java.net.DatagramSocket; +import java.net.URL; +import java.util.*; +import java.util.Map.Entry; +import java.util.concurrent.*; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + + +/** + * 客户端实例，用来管理客户端资源 + * + * @author shijia.wxr + * @since 2013-6-15 + */ +public class MQClientInstance { + private final static long LockTimeoutMillis = 3000; + private final Logger log = ClientLogger.getLog(); + private final ClientConfig clientConfig; + private final int instanceIndex; + private final String clientId; + private final long bootTimestamp = System.currentTimeMillis(); + // Producer对象 + private final ConcurrentHashMap producerTable = + new ConcurrentHashMap(); + // Consumer对象 + private final ConcurrentHashMap consumerTable = + new ConcurrentHashMap(); + // AdminExt对象 + private final ConcurrentHashMap adminExtTable = + new ConcurrentHashMap(); + // Netty客户端配置 + private final NettyClientConfig nettyClientConfig; + // RPC调用的封装类 + private final MQClientAPIImpl mQClientAPIImpl; + private final MQAdminImpl mQAdminImpl; + // 存储从Name Server拿到的Topic路由信息 + private final ConcurrentHashMap topicRouteTable = + new ConcurrentHashMap(); + // 调用Name Server获取Topic路由信息时，加锁 + private final Lock lockNamesrv = new ReentrantLock(); + // 心跳与注销动作加锁 + private final Lock lockHeartbeat = new ReentrantLock(); + // 存储Broker Name 与Broker Address的对应关系 + private final ConcurrentHashMap> brokerAddrTable = + new ConcurrentHashMap>(); + // 定时线程 + private final ScheduledExecutorService scheduledExecutorService = Executors + .newSingleThreadScheduledExecutor(new ThreadFactory() { + @Override + public Thread newThread(Runnable r) { + return new Thread(r, "MQClientFactoryScheduledThread"); + } + }); + // 处理服务器主动发来的请求 + private final ClientRemotingProcessor clientRemotingProcessor; + // 拉消息服务 + private final PullMessageService pullMessageService; + // Rebalance服务 + private final RebalanceService rebalanceService; + // 内置Producer对象 + private final DefaultMQProducer defaultMQProducer; + private ServiceState serviceState = ServiceState.CREATE_JUST; + // 监听一个UDP端口，用来防止同一个Factory启动多份（有可能分布在多个JVM中） + private DatagramSocket datagramSocket; + + private final ConsumerStatsManager consumerStatsManager; + + + public MQClientInstance(ClientConfig clientConfig, int instanceIndex, String clientId, RPCHook rpcHook) { + this.clientConfig = clientConfig; + this.instanceIndex = instanceIndex; + this.nettyClientConfig = new NettyClientConfig(); + this.nettyClientConfig.setClientCallbackExecutorThreads(clientConfig + .getClientCallbackExecutorThreads()); + this.clientRemotingProcessor = new ClientRemotingProcessor(this); + this.mQClientAPIImpl = + new MQClientAPIImpl(this.nettyClientConfig, this.clientRemotingProcessor, rpcHook); + + if (this.clientConfig.getNamesrvAddr() != null) { + this.mQClientAPIImpl.updateNameServerAddressList(this.clientConfig.getNamesrvAddr()); + log.info("user specfied name server address: {}", this.clientConfig.getNamesrvAddr()); + } + + this.clientId = clientId; + + this.mQAdminImpl = new MQAdminImpl(this); + + this.pullMessageService = new PullMessageService(this); + + this.rebalanceService = new RebalanceService(this); + + this.defaultMQProducer = new DefaultMQProducer(MixAll.CLIENT_INNER_PRODUCER_GROUP); + this.defaultMQProducer.resetClientConfig(clientConfig); + + this.consumerStatsManager = new ConsumerStatsManager(this.scheduledExecutorService); + + log.info("created a new client Instance, FactoryIndex: {} ClinetID: {} {} {}",// + this.instanceIndex, // + this.clientId, // + this.clientConfig, // + MQVersion.getVersionDesc(MQVersion.CurrentVersion)); + } + + + public MQClientInstance(ClientConfig clientConfig, int instanceIndex, String clientId) { + this(clientConfig, instanceIndex, clientId, null); + } + + + public void start() throws MQClientException { + synchronized (this) { + switch (this.serviceState) { + case CREATE_JUST: + this.serviceState = ServiceState.START_FAILED; + if (null == this.clientConfig.getNamesrvAddr()) { + this.clientConfig.setNamesrvAddr(this.mQClientAPIImpl.fetchNameServerAddr()); + } + + this.mQClientAPIImpl.start(); + this.startScheduledTask(); + this.pullMessageService.start(); + this.rebalanceService.start(); + + this.defaultMQProducer.getDefaultMQProducerImpl().start(false); + log.info("the client factory [{}] start OK", this.clientId); + this.serviceState = ServiceState.RUNNING; + break; + case RUNNING: + break; + case SHUTDOWN_ALREADY: + break; + case START_FAILED: + throw new MQClientException("The Factory object[" + this.getClientId() + + "] has been created before, and failed.", null); + default: + break; + } + } + } + + + private void startScheduledTask() { + // 定时获取Name Server地址 + if (null == this.clientConfig.getNamesrvAddr()) { + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + + @Override + public void run() { + try { + MQClientInstance.this.mQClientAPIImpl.fetchNameServerAddr(); + } + catch (Exception e) { + log.error("ScheduledTask fetchNameServerAddr exception", e); + } + } + }, 1000 * 10, 1000 * 60 * 2, TimeUnit.MILLISECONDS); + } + + // 定时从Name Server获取Topic路由信息 + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + + @Override + public void run() { + try { + MQClientInstance.this.updateTopicRouteInfoFromNameServer(); + } + catch (Exception e) { + log.error("ScheduledTask updateTopicRouteInfoFromNameServer exception", e); + } + } + }, 10, this.clientConfig.getPollNameServerInteval(), TimeUnit.MILLISECONDS); + + // 定时清理下线的Broker + // 向所有Broker发送心跳信息（包含订阅关系等） + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + + @Override + public void run() { + try { + MQClientInstance.this.cleanOfflineBroker(); + MQClientInstance.this.sendHeartbeatToAllBrokerWithLock(); + } + catch (Exception e) { + log.error("ScheduledTask sendHeartbeatToAllBroker exception", e); + } + } + }, 1000, this.clientConfig.getHeartbeatBrokerInterval(), TimeUnit.MILLISECONDS); + + // 定时持久化Consumer消费进度（广播存储到本地，集群存储到Broker） + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + + @Override + public void run() { + try { + MQClientInstance.this.persistAllConsumerOffset(); + } + catch (Exception e) { + log.error("ScheduledTask persistAllConsumerOffset exception", e); + } + } + }, 1000 * 10, this.clientConfig.getPersistConsumerOffsetInterval(), TimeUnit.MILLISECONDS); + + // 动态调整消费线程池 + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + + @Override + public void run() { + try { + MQClientInstance.this.adjustThreadPool(); + } + catch (Exception e) { + log.error("ScheduledTask adjustThreadPool exception", e); + } + } + }, 1, 1, TimeUnit.MINUTES); + } + + + /** + * 清理下线的broker + */ + private void cleanOfflineBroker() { + try { + if (this.lockNamesrv.tryLock(LockTimeoutMillis, TimeUnit.MILLISECONDS)) + try { + ConcurrentHashMap> updatedTable = + new ConcurrentHashMap>(); + + Iterator>> itBrokerTable = + this.brokerAddrTable.entrySet().iterator(); + while (itBrokerTable.hasNext()) { + Entry> entry = itBrokerTable.next(); + String brokerName = entry.getKey(); + HashMap oneTable = entry.getValue(); + + HashMap cloneAddrTable = new HashMap(); + cloneAddrTable.putAll(oneTable); + + Iterator> it = cloneAddrTable.entrySet().iterator(); + while (it.hasNext()) { + Entry ee = it.next(); + String addr = ee.getValue(); + if (!this.isBrokerAddrExistInTopicRouteTable(addr)) { + it.remove(); + log.info("the broker addr[{} {}] is offline, remove it", brokerName, addr); + } + } + + if (cloneAddrTable.isEmpty()) { + itBrokerTable.remove(); + log.info("the broker[{}] name's host is offline, remove it", brokerName); + } + else { + updatedTable.put(brokerName, cloneAddrTable); + } + } + + if (!updatedTable.isEmpty()) { + this.brokerAddrTable.putAll(updatedTable); + } + } + finally { + this.lockNamesrv.unlock(); + } + } + catch (InterruptedException e) { + log.warn("cleanOfflineBroker Exception", e); + } + } + + + private boolean isBrokerAddrExistInTopicRouteTable(final String addr) { + Iterator> it = this.topicRouteTable.entrySet().iterator(); + while (it.hasNext()) { + Entry entry = it.next(); + TopicRouteData topicRouteData = entry.getValue(); + List bds = topicRouteData.getBrokerDatas(); + for (BrokerData bd : bds) { + if (bd.getBrokerAddrs() != null) { + boolean exist = bd.getBrokerAddrs().containsValue(addr); + if (exist) + return true; + } + } + } + + return false; + } + + + private void persistAllConsumerOffset() { + Iterator> it = this.consumerTable.entrySet().iterator(); + while (it.hasNext()) { + Entry entry = it.next(); + MQConsumerInner impl = entry.getValue(); + impl.persistConsumerOffset(); + } + } + + + public void sendHeartbeatToAllBrokerWithLock() { + if (this.lockHeartbeat.tryLock()) { + try { + this.sendHeartbeatToAllBroker(); + this.uploadFilterClassSource(); + } + catch (final Exception e) { + log.error("sendHeartbeatToAllBroker exception", e); + } + finally { + this.lockHeartbeat.unlock(); + } + } + else { + log.warn("lock heartBeat, but failed."); + } + } + + + private void uploadFilterClassToAllFilterServer(final String consumerGroup, final String className, + final String topic) throws UnsupportedEncodingException { + URL classFile = FilterAPI.classFile(className); + byte[] classBody = null; + int classCRC = 0; + try { + String fileContent = MixAll.file2String(classFile); + classBody = fileContent.getBytes(MixAll.DEFAULT_CHARSET); + classCRC = UtilAll.crc32(classBody); + } + catch (Exception e1) { + log.warn("uploadFilterClassToAllFilterServer Exception, ClassFile: {} ClassName: {} {}", // + classFile,// + className,// + RemotingHelper.exceptionSimpleDesc(e1)); + } + + TopicRouteData topicRouteData = this.topicRouteTable.get(topic); + if (topicRouteData != null // + && topicRouteData.getFilterServerTable() != null + && !topicRouteData.getFilterServerTable().isEmpty()) { + Iterator>> it = + topicRouteData.getFilterServerTable().entrySet().iterator(); + while (it.hasNext()) { + Entry> next = it.next(); + List value = next.getValue(); + for (final String fsAddr : value) { + try { + this.mQClientAPIImpl.registerMessageFilterClass(fsAddr, consumerGroup, topic, + className, classCRC, classBody, 5000); + + log.info( + "register message class filter to {} OK, ConsumerGroup: {} Topic: {} ClassName: {} ClassFile: {}", + fsAddr, consumerGroup, topic, className, classFile); + + } + catch (Exception e) { + log.error("uploadFilterClassToAllFilterServer Exception", e); + } + } + } + } + else { + log.warn( + "register message class filter failed, because no filter server, ConsumerGroup: {} Topic: {} ClassName: {}", + consumerGroup, topic, className); + } + } + + + private void uploadFilterClassSource() { + Iterator> it = this.consumerTable.entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + MQConsumerInner consumer = next.getValue(); + // 只支持PushConsumer + if (ConsumeType.CONSUME_PASSIVELY == consumer.consumeType()) { + Set subscriptions = consumer.subscriptions(); + for (SubscriptionData sub : subscriptions) { + if (sub.isClassFilterMode()) { + final String consumerGroup = consumer.groupName(); + final String className = sub.getSubString(); + final String topic = sub.getTopic(); + try { + this.uploadFilterClassToAllFilterServer(consumerGroup, className, topic); + } + catch (Exception e) { + log.error("uploadFilterClassToAllFilterServer Exception", e); + } + } + } + } + } + } + + + private void sendHeartbeatToAllBroker() { + final HeartbeatData heartbeatData = this.prepareHeartbeatData(); + final boolean producerEmpty = heartbeatData.getProducerDataSet().isEmpty(); + final boolean consumerEmpty = heartbeatData.getConsumerDataSet().isEmpty(); + if (producerEmpty && consumerEmpty) { + log.warn("sending hearbeat, but no consumer and no producer"); + return; + } + + Iterator>> it = this.brokerAddrTable.entrySet().iterator(); + while (it.hasNext()) { + Entry> entry = it.next(); + String brokerName = entry.getKey(); + HashMap oneTable = entry.getValue(); + if (oneTable != null) { + for (Long id : oneTable.keySet()) { + String addr = oneTable.get(id); + if (addr != null) { + // 说明只有Producer，则不向Slave发心跳 + if (consumerEmpty) { + if (id != MixAll.MASTER_ID) + continue; + } + + try { + this.mQClientAPIImpl.sendHearbeat(addr, heartbeatData, 3000); + log.info("send heart beat to broker[{} {} {}] success", brokerName, id, addr); + log.info(heartbeatData.toString()); + } + catch (Exception e) { + log.error("send heart beat to broker exception", e); + } + } + } + } + } + } + + + private HeartbeatData prepareHeartbeatData() { + HeartbeatData heartbeatData = new HeartbeatData(); + + // clientID + heartbeatData.setClientID(this.clientId); + + // Consumer + for (String group : this.consumerTable.keySet()) { + MQConsumerInner impl = this.consumerTable.get(group); + if (impl != null) { + ConsumerData consumerData = new ConsumerData(); + consumerData.setGroupName(impl.groupName()); + consumerData.setConsumeType(impl.consumeType()); + consumerData.setMessageModel(impl.messageModel()); + consumerData.setConsumeFromWhere(impl.consumeFromWhere()); + consumerData.getSubscriptionDataSet().addAll(impl.subscriptions()); + consumerData.setUnitMode(impl.isUnitMode()); + + heartbeatData.getConsumerDataSet().add(consumerData); + } + } + + // Producer + for (String group : this.producerTable.keySet()) { + MQProducerInner impl = this.producerTable.get(group); + if (impl != null) { + ProducerData producerData = new ProducerData(); + producerData.setGroupName(group); + + heartbeatData.getProducerDataSet().add(producerData); + } + } + + return heartbeatData; + } + + + public void updateTopicRouteInfoFromNameServer() { + Set topicList = new HashSet(); + + // Consumer对象 + { + Iterator> it = this.consumerTable.entrySet().iterator(); + while (it.hasNext()) { + Entry entry = it.next(); + MQConsumerInner impl = entry.getValue(); + if (impl != null) { + Set subList = impl.subscriptions(); + if (subList != null) { + for (SubscriptionData subData : subList) { + topicList.add(subData.getTopic()); + } + } + } + } + } + + // Producer + { + Iterator> it = this.producerTable.entrySet().iterator(); + while (it.hasNext()) { + Entry entry = it.next(); + MQProducerInner impl = entry.getValue(); + if (impl != null) { + Set lst = impl.getPublishTopicList(); + topicList.addAll(lst); + } + } + } + + for (String topic : topicList) { + this.updateTopicRouteInfoFromNameServer(topic); + } + } + + + public boolean updateTopicRouteInfoFromNameServer(final String topic) { + return updateTopicRouteInfoFromNameServer(topic, false, null); + } + + + /** + * 调用Name Server接口，根据Topic获取路由信息 + */ + public boolean updateTopicRouteInfoFromNameServer(final String topic, boolean isDefault, + DefaultMQProducer defaultMQProducer) { + try { + if (this.lockNamesrv.tryLock(LockTimeoutMillis, TimeUnit.MILLISECONDS)) { + try { + TopicRouteData topicRouteData; + if (isDefault && defaultMQProducer != null) { + topicRouteData = + this.mQClientAPIImpl.getDefaultTopicRouteInfoFromNameServer( + defaultMQProducer.getCreateTopicKey(), 1000 * 3); + if (topicRouteData != null) { + for (QueueData data : topicRouteData.getQueueDatas()) { + // 读写分区个数是一致，故只做一次判断 + int queueNums = + Math.min(defaultMQProducer.getDefaultTopicQueueNums(), + data.getReadQueueNums()); + data.setReadQueueNums(queueNums); + data.setWriteQueueNums(queueNums); + } + } + } + else { + topicRouteData = + this.mQClientAPIImpl.getTopicRouteInfoFromNameServer(topic, 1000 * 3); + } + if (topicRouteData != null) { + TopicRouteData old = this.topicRouteTable.get(topic); + boolean changed = topicRouteDataIsChange(old, topicRouteData); + if (!changed) { + changed = this.isNeedUpdateTopicRouteInfo(topic); + } + else { + log.info("the topic[{}] route info changed, odl[{}] ,new[{}]", topic, old, + topicRouteData); + } + + if (changed) { + // 后面排序会影响下次的equal逻辑判断，所以先clone一份 + TopicRouteData cloneTopicRouteData = topicRouteData.cloneTopicRouteData(); + + // 更新Broker地址信息 + for (BrokerData bd : topicRouteData.getBrokerDatas()) { + this.brokerAddrTable.put(bd.getBrokerName(), bd.getBrokerAddrs()); + } + + // 更新发布队列信息 + { + TopicPublishInfo publishInfo = + topicRouteData2TopicPublishInfo(topic, topicRouteData); + publishInfo.setHaveTopicRouterInfo(true); + Iterator> it = + this.producerTable.entrySet().iterator(); + while (it.hasNext()) { + Entry entry = it.next(); + MQProducerInner impl = entry.getValue(); + if (impl != null) { + impl.updateTopicPublishInfo(topic, publishInfo); + } + } + } + + // 更新订阅队列信息 + { + Set subscribeInfo = + topicRouteData2TopicSubscribeInfo(topic, topicRouteData); + Iterator> it = + this.consumerTable.entrySet().iterator(); + while (it.hasNext()) { + Entry entry = it.next(); + MQConsumerInner impl = entry.getValue(); + if (impl != null) { + impl.updateTopicSubscribeInfo(topic, subscribeInfo); + } + } + } + log.info("topicRouteTable.put TopicRouteData[{}]", cloneTopicRouteData); + this.topicRouteTable.put(topic, cloneTopicRouteData); + return true; + } + } + else { + log.warn( + "updateTopicRouteInfoFromNameServer, getTopicRouteInfoFromNameServer return null, Topic: {}", + topic); + } + } + catch (Exception e) { + if (!topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX) + && !topic.equals(MixAll.DEFAULT_TOPIC)) { + log.warn("updateTopicRouteInfoFromNameServer Exception", e); + } + } + finally { + this.lockNamesrv.unlock(); + } + } + else { + log.warn("updateTopicRouteInfoFromNameServer tryLock timeout {}ms", LockTimeoutMillis); + } + } + catch (InterruptedException e) { + log.warn("updateTopicRouteInfoFromNameServer Exception", e); + } + + return false; + } + + + private boolean topicRouteDataIsChange(TopicRouteData olddata, TopicRouteData nowdata) { + if (olddata == null || nowdata == null) + return true; + TopicRouteData old = olddata.cloneTopicRouteData(); + TopicRouteData now = nowdata.cloneTopicRouteData(); + Collections.sort(old.getQueueDatas()); + Collections.sort(old.getBrokerDatas()); + Collections.sort(now.getQueueDatas()); + Collections.sort(now.getBrokerDatas()); + return !old.equals(now); + + } + + + public static TopicPublishInfo topicRouteData2TopicPublishInfo(final String topic, + final TopicRouteData route) { + TopicPublishInfo info = new TopicPublishInfo(); + // 顺序消息 + if (route.getOrderTopicConf() != null && route.getOrderTopicConf().length() > 0) { + String[] brokers = route.getOrderTopicConf().split(";"); + for (String broker : brokers) { + String[] item = broker.split(":"); + int nums = Integer.parseInt(item[1]); + for (int i = 0; i < nums; i++) { + MessageQueue mq = new MessageQueue(topic, item[0], i); + info.getMessageQueueList().add(mq); + } + } + + info.setOrderTopic(true); + } + // 非顺序消息 + else { + List qds = route.getQueueDatas(); + // 排序原因：即使没有配置顺序消息模式，默认队列的顺序同配置的一致。 + Collections.sort(qds); + for (QueueData qd : qds) { + if (PermName.isWriteable(qd.getPerm())) { + // 这里需要判断BrokerName对应的Master是否存在，因为只能向Master发送消息 + BrokerData brokerData = null; + for (BrokerData bd : route.getBrokerDatas()) { + if (bd.getBrokerName().equals(qd.getBrokerName())) { + brokerData = bd; + break; + } + } + + if (null == brokerData) { + continue; + } + + if (!brokerData.getBrokerAddrs().containsKey(MixAll.MASTER_ID)) { + continue; + } + + for (int i = 0; i < qd.getWriteQueueNums(); i++) { + MessageQueue mq = new MessageQueue(topic, qd.getBrokerName(), i); + info.getMessageQueueList().add(mq); + } + } + } + + info.setOrderTopic(false); + } + + return info; + } + + + public static Set topicRouteData2TopicSubscribeInfo(final String topic, + final TopicRouteData route) { + Set mqList = new HashSet(); + List qds = route.getQueueDatas(); + for (QueueData qd : qds) { + if (PermName.isReadable(qd.getPerm())) { + for (int i = 0; i < qd.getReadQueueNums(); i++) { + MessageQueue mq = new MessageQueue(topic, qd.getBrokerName(), i); + mqList.add(mq); + } + } + } + + return mqList; + } + + + private boolean isNeedUpdateTopicRouteInfo(final String topic) { + boolean result = false; + // 查看发布队列是否需要更新 + { + Iterator> it = this.producerTable.entrySet().iterator(); + while (it.hasNext() && !result) { + Entry entry = it.next(); + MQProducerInner impl = entry.getValue(); + if (impl != null) { + result = impl.isPublishTopicNeedUpdate(topic); + } + } + } + + // 查看订阅队列是否需要更新 + { + Iterator> it = this.consumerTable.entrySet().iterator(); + while (it.hasNext() && !result) { + Entry entry = it.next(); + MQConsumerInner impl = entry.getValue(); + if (impl != null) { + result = impl.isSubscribeTopicNeedUpdate(topic); + } + } + } + + return result; + } + + + public void shutdown() { + // Consumer + if (!this.consumerTable.isEmpty()) + return; + + // AdminExt + if (!this.adminExtTable.isEmpty()) + return; + + // Producer + if (this.producerTable.size() > 1) + return; + + synchronized (this) { + switch (this.serviceState) { + case CREATE_JUST: + break; + case RUNNING: + this.defaultMQProducer.getDefaultMQProducerImpl().shutdown(false); + + this.serviceState = ServiceState.SHUTDOWN_ALREADY; + this.pullMessageService.shutdown(true); + this.scheduledExecutorService.shutdown(); + this.mQClientAPIImpl.shutdown(); + this.rebalanceService.shutdown(); + + if (this.datagramSocket != null) { + this.datagramSocket.close(); + this.datagramSocket = null; + } + MQClientManager.getInstance().removeClientFactory(this.clientId); + log.info("the client factory [{}] shutdown OK", this.clientId); + break; + case SHUTDOWN_ALREADY: + break; + default: + break; + } + } + } + + + public boolean registerConsumer(final String group, final MQConsumerInner consumer) { + if (null == group || null == consumer) { + return false; + } + + MQConsumerInner prev = this.consumerTable.putIfAbsent(group, consumer); + if (prev != null) { + log.warn("the consumer group[" + group + "] exist already."); + return false; + } + + return true; + } + + + public void unregisterConsumer(final String group) { + this.consumerTable.remove(group); + this.unregisterClientWithLock(null, group); + } + + + private void unregisterClientWithLock(final String producerGroup, final String consumerGroup) { + try { + if (this.lockHeartbeat.tryLock(LockTimeoutMillis, TimeUnit.MILLISECONDS)) { + try { + this.unregisterClient(producerGroup, consumerGroup); + } + catch (Exception e) { + log.error("unregisterClient exception", e); + } + finally { + this.lockHeartbeat.unlock(); + } + } + else { + log.warn("lock heartBeat, but failed."); + } + } + catch (InterruptedException e) { + log.warn("unregisterClientWithLock exception", e); + } + } + + + private void unregisterClient(final String producerGroup, final String consumerGroup) { + Iterator>> it = this.brokerAddrTable.entrySet().iterator(); + while (it.hasNext()) { + Entry> entry = it.next(); + String brokerName = entry.getKey(); + HashMap oneTable = entry.getValue(); + + if (oneTable != null) { + for (Long id : oneTable.keySet()) { + String addr = oneTable.get(id); + if (addr != null) { + try { + this.mQClientAPIImpl.unregisterClient(addr, this.clientId, producerGroup, + consumerGroup, 3000); + log.info( + "unregister client[Producer: {} Consumer: {}] from broker[{} {} {}] success", + producerGroup, consumerGroup, brokerName, id, addr); + } + catch (RemotingException e) { + log.error("unregister client exception from broker: " + addr, e); + } + catch (MQBrokerException e) { + log.error("unregister client exception from broker: " + addr, e); + } + catch (InterruptedException e) { + log.error("unregister client exception from broker: " + addr, e); + } + } + } + } + } + } + + + public boolean registerProducer(final String group, final DefaultMQProducerImpl producer) { + if (null == group || null == producer) { + return false; + } + + MQProducerInner prev = this.producerTable.putIfAbsent(group, producer); + if (prev != null) { + log.warn("the producer group[{}] exist already.", group); + return false; + } + + return true; + } + + + public void unregisterProducer(final String group) { + this.producerTable.remove(group); + this.unregisterClientWithLock(group, null); + } + + + public boolean registerAdminExt(final String group, final MQAdminExtInner admin) { + if (null == group || null == admin) { + return false; + } + + MQAdminExtInner prev = this.adminExtTable.putIfAbsent(group, admin); + if (prev != null) { + log.warn("the admin group[{}] exist already.", group); + return false; + } + + return true; + } + + + public void unregisterAdminExt(final String group) { + this.adminExtTable.remove(group); + } + + + public void rebalanceImmediately() { + this.rebalanceService.wakeup(); + } + + + public void doRebalance() { + for (String group : this.consumerTable.keySet()) { + MQConsumerInner impl = this.consumerTable.get(group); + if (impl != null) { + try { + impl.doRebalance(); + } + catch (Exception e) { + log.error("doRebalance exception", e); + } + } + } + } + + + public MQProducerInner selectProducer(final String group) { + return this.producerTable.get(group); + } + + + public MQConsumerInner selectConsumer(final String group) { + return this.consumerTable.get(group); + } + + + /** + * 管理类的接口查询Broker地址，Master优先 + * + * @param brokerName + * @return + */ + public FindBrokerResult findBrokerAddressInAdmin(final String brokerName) { + String brokerAddr = null; + boolean slave = false; + boolean found = false; + + HashMap map = this.brokerAddrTable.get(brokerName); + if (map != null && !map.isEmpty()) { + // TODO 如果有多个Slave，可能会每次都选中相同的Slave，这里需要优化 + FOR_SEG: for (Map.Entry entry : map.entrySet()) { + Long id = entry.getKey(); + brokerAddr = entry.getValue(); + if (brokerAddr != null) { + found = true; + if (MixAll.MASTER_ID == id) { + slave = false; + break FOR_SEG; + } + else { + slave = true; + } + break; + + } + } // end of for + } + + if (found) { + return new FindBrokerResult(brokerAddr, slave); + } + + return null; + } + + + /** + * 发布消息过程中，寻找Broker地址，一定是找Master + */ + public String findBrokerAddressInPublish(final String brokerName) { + HashMap map = this.brokerAddrTable.get(brokerName); + if (map != null && !map.isEmpty()) { + return map.get(MixAll.MASTER_ID); + } + + return null; + } + + + /** + * 订阅消息过程中，寻找Broker地址，取Master还是Slave由参数决定 + */ + public FindBrokerResult findBrokerAddressInSubscribe(// + final String brokerName,// + final long brokerId,// + final boolean onlyThisBroker// + ) { + String brokerAddr = null; + boolean slave = false; + boolean found = false; + + HashMap map = this.brokerAddrTable.get(brokerName); + if (map != null && !map.isEmpty()) { + brokerAddr = map.get(brokerId); + slave = (brokerId != MixAll.MASTER_ID); + found = (brokerAddr != null); + + // 尝试寻找其他Broker + if (!found && !onlyThisBroker) { + Entry entry = map.entrySet().iterator().next(); + brokerAddr = entry.getValue(); + slave = (entry.getKey() != MixAll.MASTER_ID); + found = true; + } + } + + if (found) { + return new FindBrokerResult(brokerAddr, slave); + } + + return null; + } + + + public List findConsumerIdList(final String topic, final String group) { + String brokerAddr = this.findBrokerAddrByTopic(topic); + if (null == brokerAddr) { + this.updateTopicRouteInfoFromNameServer(topic); + brokerAddr = this.findBrokerAddrByTopic(topic); + } + + if (null != brokerAddr) { + try { + return this.mQClientAPIImpl.getConsumerIdListByGroup(brokerAddr, group, 3000); + } + catch (Exception e) { + log.warn("getConsumerIdListByGroup exception, " + brokerAddr + " " + group, e); + } + } + + return null; + } + + + public String findBrokerAddrByTopic(final String topic) { + TopicRouteData topicRouteData = this.topicRouteTable.get(topic); + if (topicRouteData != null) { + List brokers = topicRouteData.getBrokerDatas(); + if (!brokers.isEmpty()) { + BrokerData bd = brokers.get(0); + return bd.selectBrokerAddr(); + } + } + + return null; + } + + + public void resetOffset(String topic, String group, Map offsetTable) { + DefaultMQPushConsumerImpl consumer = null; + try { + MQConsumerInner impl = this.consumerTable.get(group); + if (impl != null && impl instanceof DefaultMQPushConsumerImpl) { + consumer = (DefaultMQPushConsumerImpl) impl; + } + else { + log.info("[reset-offset] consumer dose not exist. group={}", group); + return; + } + + // 设置当前的 processQueue 为 dropped，从而使得当前的 pullRequest 以及 + // consumerRequest 处理结束并销毁 + ConcurrentHashMap processQueueTable = + consumer.getRebalanceImpl().getProcessQueueTable(); + Iterator itr = processQueueTable.keySet().iterator(); + while (itr.hasNext()) { + MessageQueue mq = itr.next(); + if (topic.equals(mq.getTopic())) { + ProcessQueue pq = processQueueTable.get(mq); + pq.setDroped(true); + pq.clear(); + } + } + + // 更新消费队列的 offset 并提交到 broker + Iterator iterator = offsetTable.keySet().iterator(); + while (iterator.hasNext()) { + MessageQueue mq = iterator.next(); + consumer.updateConsumeOffset(mq, offsetTable.get(mq)); + log.info("[reset-offset] reset offsetTable. topic={}, group={}, mq={}, offset={}", + new Object[] { topic, group, mq, offsetTable.get(mq) }); + } + consumer.getOffsetStore().persistAll(offsetTable.keySet()); + + // 等待所有的 pullRequest 以及 consumerRequest 处理完成 + try { + TimeUnit.SECONDS.sleep(10); + } + catch (InterruptedException e) { + // + } + + // 更新消费队列的 offset 并提交到 broker + iterator = offsetTable.keySet().iterator(); + while (iterator.hasNext()) { + MessageQueue mq = iterator.next(); + consumer.updateConsumeOffset(mq, offsetTable.get(mq)); + log.info("[reset-offset] reset offsetTable. topic={}, group={}, mq={}, offset={}", + new Object[] { topic, group, mq, offsetTable.get(mq) }); + } + consumer.getOffsetStore().persistAll(offsetTable.keySet()); + + // 真正清除被 dropped 的 processQueue，从而使得新的 rebalance 生效，生成新的 pullRequest + // 以及 consumerRequest + iterator = offsetTable.keySet().iterator(); + processQueueTable = consumer.getRebalanceImpl().getProcessQueueTable(); + while (iterator.hasNext()) { + MessageQueue mq = iterator.next(); + processQueueTable.remove(mq); + } + } + finally { + // 放在 finally 主要是确保 rebalance 一定被执行 + consumer.getRebalanceImpl().doRebalance(); + } + } + + + public Map getConsumerStatus(String topic, String group) { + MQConsumerInner impl = this.consumerTable.get(group); + if (impl != null && impl instanceof DefaultMQPushConsumerImpl) { + DefaultMQPushConsumerImpl consumer = (DefaultMQPushConsumerImpl) impl; + return consumer.getOffsetStore().cloneOffsetTable(topic); + } + else if (impl != null && impl instanceof DefaultMQPullConsumerImpl) { + DefaultMQPullConsumerImpl consumer = (DefaultMQPullConsumerImpl) impl; + return consumer.getOffsetStore().cloneOffsetTable(topic); + } + else { + return Collections.EMPTY_MAP; + } + } + + + public TopicRouteData getAnExistTopicRouteData(final String topic) { + return this.topicRouteTable.get(topic); + } + + + public MQClientAPIImpl getMQClientAPIImpl() { + return mQClientAPIImpl; + } + + + public MQAdminImpl getMQAdminImpl() { + return mQAdminImpl; + } + + + public String getClientId() { + return clientId; + } + + + public long getBootTimestamp() { + return bootTimestamp; + } + + + public ScheduledExecutorService getScheduledExecutorService() { + return scheduledExecutorService; + } + + + public PullMessageService getPullMessageService() { + return pullMessageService; + } + + + public DefaultMQProducer getDefaultMQProducer() { + return defaultMQProducer; + } + + + public ConcurrentHashMap getTopicRouteTable() { + return topicRouteTable; + } + + + public void adjustThreadPool() { + Iterator> it = this.consumerTable.entrySet().iterator(); + while (it.hasNext()) { + Entry entry = it.next(); + MQConsumerInner impl = entry.getValue(); + if (impl != null) { + try { + if (impl instanceof DefaultMQPushConsumerImpl) { + DefaultMQPushConsumerImpl dmq = (DefaultMQPushConsumerImpl) impl; + dmq.adjustThreadPool(); + } + } + catch (Exception e) { + } + } + } + } + + + public ConsumeMessageDirectlyResult consumeMessageDirectly(final MessageExt msg, // + final String consumerGroup, // + final String brokerName) { + MQConsumerInner mqConsumerInner = this.consumerTable.get(consumerGroup); + if (null != mqConsumerInner) { + DefaultMQPushConsumerImpl consumer = (DefaultMQPushConsumerImpl) mqConsumerInner; + + ConsumeMessageDirectlyResult result = + consumer.getConsumeMessageService().consumeMessageDirectly(msg, brokerName); + return result; + } + + return null; + } + + + public ConsumerRunningInfo consumerRunningInfo(final String consumerGroup) { + MQConsumerInner mqConsumerInner = this.consumerTable.get(consumerGroup); + + ConsumerRunningInfo consumerRunningInfo = mqConsumerInner.consumerRunningInfo(); + + // 补充额外的信息 + List nsList = this.mQClientAPIImpl.getRemotingClient().getNameServerAddressList(); + String nsAddr = ""; + if (nsList != null) { + for (String addr : nsList) { + nsAddr = nsAddr + addr + ";"; + } + } + + consumerRunningInfo.getProperties().put(ConsumerRunningInfo.PROP_NAMESERVER_ADDR, nsAddr); + consumerRunningInfo.getProperties().put(ConsumerRunningInfo.PROP_CONSUME_TYPE, + mqConsumerInner.consumeType()); + consumerRunningInfo.getProperties().put(ConsumerRunningInfo.PROP_CLIENT_VERSION, + MQVersion.getVersionDesc(MQVersion.CurrentVersion)); + + return consumerRunningInfo; + } + + + public ConsumerStatsManager getConsumerStatsManager() { + return consumerStatsManager; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/producer/DefaultMQProducerImpl.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/producer/DefaultMQProducerImpl.java new file mode 100644 index 000000000..8fbba682c --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/producer/DefaultMQProducerImpl.java @@ -0,0 +1,1058 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.impl.producer; + +import java.io.IOException; +import java.net.UnknownHostException; +import java.util.*; +import java.util.concurrent.*; + +import org.slf4j.Logger; + +import com.alibaba.rocketmq.client.QueryResult; +import com.alibaba.rocketmq.client.Validators; +import com.alibaba.rocketmq.client.exception.MQBrokerException; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.client.hook.CheckForbiddenContext; +import com.alibaba.rocketmq.client.hook.CheckForbiddenHook; +import com.alibaba.rocketmq.client.hook.SendMessageContext; +import com.alibaba.rocketmq.client.hook.SendMessageHook; +import com.alibaba.rocketmq.client.impl.CommunicationMode; +import com.alibaba.rocketmq.client.impl.MQClientManager; +import com.alibaba.rocketmq.client.impl.factory.MQClientInstance; +import com.alibaba.rocketmq.client.log.ClientLogger; +import com.alibaba.rocketmq.client.producer.*; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.ServiceState; +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.help.FAQUrl; +import com.alibaba.rocketmq.common.message.*; +import com.alibaba.rocketmq.common.protocol.ResponseCode; +import com.alibaba.rocketmq.common.protocol.header.CheckTransactionStateRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.EndTransactionRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.SendMessageRequestHeader; +import com.alibaba.rocketmq.common.sysflag.MessageSysFlag; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.remoting.common.RemotingHelper; +import com.alibaba.rocketmq.remoting.common.RemotingUtil; +import com.alibaba.rocketmq.remoting.exception.RemotingException; + + +/** + * 生产者默认实现 + * + * @author shijia.wxr + * @since 2013-7-24 + */ +public class DefaultMQProducerImpl implements MQProducerInner { + private final Logger log = ClientLogger.getLog(); + private final DefaultMQProducer defaultMQProducer; + private final ConcurrentHashMap topicPublishInfoTable = + new ConcurrentHashMap(); + /** + * 事务相关 + */ + protected BlockingQueue checkRequestQueue; + protected ExecutorService checkExecutor; + private ServiceState serviceState = ServiceState.CREATE_JUST; + private MQClientInstance mQClientFactory; + + /** + * 发送每条消息会回调 + */ + private final ArrayList sendMessageHookList = new ArrayList(); + /** + * 发消息之前，读写权限控制时调用 Hook + */ + private ArrayList checkForbiddenHookList = new ArrayList(); + + /** + * 通信层hook + */ + private final RPCHook rpcHook; + + + public DefaultMQProducerImpl(final DefaultMQProducer defaultMQProducer, RPCHook rpcHook) { + this.defaultMQProducer = defaultMQProducer; + this.rpcHook = rpcHook; + } + + + public DefaultMQProducerImpl(final DefaultMQProducer defaultMQProducer) { + this(defaultMQProducer, null); + } + + + public boolean hasCheckForbiddenHook() { + return !checkForbiddenHookList.isEmpty(); + } + + + public void registerCheckForbiddenHook(CheckForbiddenHook checkForbiddenHook) { + this.checkForbiddenHookList.add(checkForbiddenHook); + log.info("register a new checkForbiddenHook. hookName={}, allHookSize={}", + checkForbiddenHook.hookName(), checkForbiddenHookList.size()); + } + + + public void executeCheckForbiddenHook(final CheckForbiddenContext context) throws MQClientException { + if (hasCheckForbiddenHook()) { + for (CheckForbiddenHook hook : checkForbiddenHookList) { + hook.checkForbidden(context); + } + } + } + + + public void initTransactionEnv() { + TransactionMQProducer producer = (TransactionMQProducer) this.defaultMQProducer; + this.checkRequestQueue = new LinkedBlockingQueue(producer.getCheckRequestHoldMax()); + this.checkExecutor = new ThreadPoolExecutor(// + producer.getCheckThreadPoolMinSize(),// + producer.getCheckThreadPoolMaxSize(),// + 1000 * 60,// + TimeUnit.MILLISECONDS,// + this.checkRequestQueue); + } + + + public void destroyTransactionEnv() { + this.checkExecutor.shutdown(); + this.checkRequestQueue.clear(); + } + + + public boolean hasSendMessageHook() { + return !this.sendMessageHookList.isEmpty(); + } + + + public void registerSendMessageHook(final SendMessageHook hook) { + this.sendMessageHookList.add(hook); + log.info("register sendMessage Hook, {}", hook.hookName()); + } + + + public void executeSendMessageHookBefore(final SendMessageContext context) { + if (!this.sendMessageHookList.isEmpty()) { + for (SendMessageHook hook : this.sendMessageHookList) { + try { + hook.sendMessageBefore(context); + } + catch (Throwable e) { + } + } + } + } + + + public void executeSendMessageHookAfter(final SendMessageContext context) { + if (!this.sendMessageHookList.isEmpty()) { + for (SendMessageHook hook : this.sendMessageHookList) { + try { + hook.sendMessageAfter(context); + } + catch (Throwable e) { + } + } + } + } + + + public void start() throws MQClientException { + this.start(true); + } + + + public void start(final boolean startFactory) throws MQClientException { + switch (this.serviceState) { + case CREATE_JUST: + this.serviceState = ServiceState.START_FAILED; + + this.checkConfig(); + + if (!this.defaultMQProducer.getProducerGroup().equals(MixAll.CLIENT_INNER_PRODUCER_GROUP)) { + this.defaultMQProducer.changeInstanceNameToPID(); + } + + this.mQClientFactory = + MQClientManager.getInstance().getAndCreateMQClientInstance(this.defaultMQProducer, + rpcHook); + + boolean registerOK = + mQClientFactory.registerProducer(this.defaultMQProducer.getProducerGroup(), this); + if (!registerOK) { + this.serviceState = ServiceState.CREATE_JUST; + throw new MQClientException("The producer group[" + this.defaultMQProducer.getProducerGroup() + + "] has been created before, specify another name please." + + FAQUrl.suggestTodo(FAQUrl.GROUP_NAME_DUPLICATE_URL), null); + } + + // 默认Topic注册 + this.topicPublishInfoTable + .put(this.defaultMQProducer.getCreateTopicKey(), new TopicPublishInfo()); + + if (startFactory) { + mQClientFactory.start(); + } + + log.info("the producer [{}] start OK", this.defaultMQProducer.getProducerGroup()); + this.serviceState = ServiceState.RUNNING; + break; + case RUNNING: + case START_FAILED: + case SHUTDOWN_ALREADY: + throw new MQClientException("The producer service state not OK, maybe started once, "// + + this.serviceState// + + FAQUrl.suggestTodo(FAQUrl.CLIENT_SERVICE_NOT_OK), null); + default: + break; + } + + this.mQClientFactory.sendHeartbeatToAllBrokerWithLock(); + } + + + private void checkConfig() throws MQClientException { + // producerGroup 有效性检查 + Validators.checkGroup(this.defaultMQProducer.getProducerGroup()); + + if (null == this.defaultMQProducer.getProducerGroup()) { + throw new MQClientException("producerGroup is null", null); + } + + if (this.defaultMQProducer.getProducerGroup().equals(MixAll.DEFAULT_PRODUCER_GROUP)) { + throw new MQClientException("producerGroup can not equal " + MixAll.DEFAULT_PRODUCER_GROUP + + ", please specify another one.", null); + } + } + + + public void shutdown() { + this.shutdown(true); + } + + + public void shutdown(final boolean shutdownFactory) { + switch (this.serviceState) { + case CREATE_JUST: + break; + case RUNNING: + this.mQClientFactory.unregisterProducer(this.defaultMQProducer.getProducerGroup()); + if (shutdownFactory) { + this.mQClientFactory.shutdown(); + } + + log.info("the producer [{}] shutdown OK", this.defaultMQProducer.getProducerGroup()); + this.serviceState = ServiceState.SHUTDOWN_ALREADY; + break; + case SHUTDOWN_ALREADY: + break; + default: + break; + } + } + + + @Override + public Set getPublishTopicList() { + Set topicList = new HashSet(); + for (String key : this.topicPublishInfoTable.keySet()) { + topicList.add(key); + } + + return topicList; + } + + + @Override + public boolean isPublishTopicNeedUpdate(String topic) { + TopicPublishInfo prev = this.topicPublishInfoTable.get(topic); + + return null == prev || !prev.ok(); + } + + + @Override + public TransactionCheckListener checkListener() { + if (this.defaultMQProducer instanceof TransactionMQProducer) { + TransactionMQProducer producer = (TransactionMQProducer) defaultMQProducer; + return producer.getTransactionCheckListener(); + } + + return null; + } + + + @Override + public void checkTransactionState(final String addr, final MessageExt msg, + final CheckTransactionStateRequestHeader header) { + Runnable request = new Runnable() { + private final String brokerAddr = addr; + private final MessageExt message = msg; + private final CheckTransactionStateRequestHeader checkRequestHeader = header; + private final String group = DefaultMQProducerImpl.this.defaultMQProducer.getProducerGroup(); + + + @Override + public void run() { + TransactionCheckListener transactionCheckListener = + DefaultMQProducerImpl.this.checkListener(); + if (transactionCheckListener != null) { + LocalTransactionState localTransactionState = LocalTransactionState.UNKNOW; + Throwable exception = null; + try { + localTransactionState = transactionCheckListener.checkLocalTransactionState(message); + } + catch (Throwable e) { + log.error( + "Broker call checkTransactionState, but checkLocalTransactionState exception", e); + exception = e; + } + + this.processTransactionState(// + localTransactionState,// + group, // + exception); + } + else { + log.warn("checkTransactionState, pick transactionCheckListener by group[{}] failed", + group); + } + } + + + private void processTransactionState(// + final LocalTransactionState localTransactionState,// + final String producerGroup,// + final Throwable exception) { + final EndTransactionRequestHeader thisHeader = new EndTransactionRequestHeader(); + thisHeader.setCommitLogOffset(checkRequestHeader.getCommitLogOffset()); + thisHeader.setProducerGroup(producerGroup); + thisHeader.setTranStateTableOffset(checkRequestHeader.getTranStateTableOffset()); + thisHeader.setFromTransactionCheck(true); + thisHeader.setMsgId(message.getMsgId()); + switch (localTransactionState) { + case COMMIT_MESSAGE: + thisHeader.setCommitOrRollback(MessageSysFlag.TransactionCommitType); + break; + case ROLLBACK_MESSAGE: + thisHeader.setCommitOrRollback(MessageSysFlag.TransactionRollbackType); + log.warn("when broker check, client rollback this transaction, {}", thisHeader); + break; + case UNKNOW: + thisHeader.setCommitOrRollback(MessageSysFlag.TransactionNotType); + log.warn("when broker check, client donot know this transaction state, {}", thisHeader); + break; + default: + break; + } + + String remark = null; + if (exception != null) { + remark = + "checkLocalTransactionState Exception: " + + RemotingHelper.exceptionSimpleDesc(exception); + } + + try { + DefaultMQProducerImpl.this.mQClientFactory.getMQClientAPIImpl().endTransactionOneway( + brokerAddr, thisHeader, remark, 3000); + } + catch (Exception e) { + log.error("endTransactionOneway exception", e); + } + } + }; + + this.checkExecutor.submit(request); + } + + + @Override + public void updateTopicPublishInfo(final String topic, final TopicPublishInfo info) { + if (info != null && topic != null) { + TopicPublishInfo prev = this.topicPublishInfoTable.put(topic, info); + if (prev != null) { + info.getSendWhichQueue().set(prev.getSendWhichQueue().get()); + log.info("updateTopicPublishInfo prev is not null, " + prev.toString()); + } + } + } + + + @Override + public boolean isUnitMode() { + return this.defaultMQProducer.isUnitMode(); + } + + + public void createTopic(String key, String newTopic, int queueNum) throws MQClientException { + createTopic(key, newTopic, queueNum, 0); + } + + + public void createTopic(String key, String newTopic, int queueNum, int topicSysFlag) + throws MQClientException { + this.makeSureStateOK(); + // topic 有效性检查 + Validators.checkTopic(newTopic); + + this.mQClientFactory.getMQAdminImpl().createTopic(key, newTopic, queueNum, topicSysFlag); + } + + + private void makeSureStateOK() throws MQClientException { + if (this.serviceState != ServiceState.RUNNING) { + throw new MQClientException("The producer service state not OK, "// + + this.serviceState// + + FAQUrl.suggestTodo(FAQUrl.CLIENT_SERVICE_NOT_OK), null); + } + } + + + public List fetchPublishMessageQueues(String topic) throws MQClientException { + this.makeSureStateOK(); + return this.mQClientFactory.getMQAdminImpl().fetchPublishMessageQueues(topic); + } + + + public long searchOffset(MessageQueue mq, long timestamp) throws MQClientException { + this.makeSureStateOK(); + return this.mQClientFactory.getMQAdminImpl().searchOffset(mq, timestamp); + } + + + public long maxOffset(MessageQueue mq) throws MQClientException { + this.makeSureStateOK(); + return this.mQClientFactory.getMQAdminImpl().maxOffset(mq); + } + + + public long minOffset(MessageQueue mq) throws MQClientException { + this.makeSureStateOK(); + return this.mQClientFactory.getMQAdminImpl().minOffset(mq); + } + + + public long earliestMsgStoreTime(MessageQueue mq) throws MQClientException { + this.makeSureStateOK(); + return this.mQClientFactory.getMQAdminImpl().earliestMsgStoreTime(mq); + } + + + public MessageExt viewMessage(String msgId) throws RemotingException, MQBrokerException, + InterruptedException, MQClientException { + this.makeSureStateOK(); + + return this.mQClientFactory.getMQAdminImpl().viewMessage(msgId); + } + + + public QueryResult queryMessage(String topic, String key, int maxNum, long begin, long end) + throws MQClientException, InterruptedException { + this.makeSureStateOK(); + return this.mQClientFactory.getMQAdminImpl().queryMessage(topic, key, maxNum, begin, end); + } + + + /** + * DEFAULT ASYNC ------------------------------------------------------- + */ + public void send(Message msg, SendCallback sendCallback) throws MQClientException, RemotingException, + InterruptedException { + try { + this.sendDefaultImpl(msg, CommunicationMode.ASYNC, sendCallback); + } + catch (MQBrokerException e) { + throw new MQClientException("unknow exception", e); + } + } + + + private SendResult sendDefaultImpl(// + Message msg,// + final CommunicationMode communicationMode,// + final SendCallback sendCallback// + ) throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + // 有效性检查 + this.makeSureStateOK(); + Validators.checkMessage(msg, this.defaultMQProducer); + + final long maxTimeout = this.defaultMQProducer.getSendMsgTimeout() + 1000; + final long beginTimestamp = System.currentTimeMillis(); + long endTimestamp = beginTimestamp; + TopicPublishInfo topicPublishInfo = this.tryToFindTopicPublishInfo(msg.getTopic()); + if (topicPublishInfo != null && topicPublishInfo.ok()) { + MessageQueue mq = null; + Exception exception = null; + SendResult sendResult = null; + int timesTotal = 1 + this.defaultMQProducer.getRetryTimesWhenSendFailed(); + int times = 0; + // 记录投递的BrokerName + String[] brokersSent = new String[timesTotal]; + for (; times < timesTotal && (endTimestamp - beginTimestamp) < maxTimeout; times++) { + String lastBrokerName = null == mq ? null : mq.getBrokerName(); + MessageQueue tmpmq = topicPublishInfo.selectOneMessageQueue(lastBrokerName); + if (tmpmq != null) { + mq = tmpmq; + brokersSent[times] = mq.getBrokerName(); + try { + sendResult = this.sendKernelImpl(msg, mq, communicationMode, sendCallback); + endTimestamp = System.currentTimeMillis(); + switch (communicationMode) { + case ASYNC: + return null; + case ONEWAY: + return null; + case SYNC: + if (sendResult.getSendStatus() != SendStatus.SEND_OK) { + if (this.defaultMQProducer.isRetryAnotherBrokerWhenNotStoreOK()) { + continue; + } + } + + return sendResult; + default: + break; + } + } + catch (RemotingException e) { + log.warn("sendKernelImpl exception", e); + log.warn(msg.toString()); + exception = e; + endTimestamp = System.currentTimeMillis(); + continue; + } + catch (MQClientException e) { + log.warn("sendKernelImpl exception", e); + log.warn(msg.toString()); + exception = e; + endTimestamp = System.currentTimeMillis(); + continue; + } + catch (MQBrokerException e) { + log.warn("sendKernelImpl exception", e); + log.warn(msg.toString()); + exception = e; + endTimestamp = System.currentTimeMillis(); + switch (e.getResponseCode()) { + case ResponseCode.TOPIC_NOT_EXIST: + case ResponseCode.SERVICE_NOT_AVAILABLE: + case ResponseCode.SYSTEM_ERROR: + case ResponseCode.NO_PERMISSION: + case ResponseCode.NO_BUYER_ID: + case ResponseCode.NOT_IN_CURRENT_UNIT: + continue; + default: + if (sendResult != null) { + return sendResult; + } + + throw e; + } + } + catch (InterruptedException e) { + log.warn("sendKernelImpl exception", e); + log.warn(msg.toString()); + throw e; + } + } + else { + break; + } + } // end of for + + if (sendResult != null) { + return sendResult; + } + + String info = + String.format("Send [%d] times, still failed, cost [%d]ms, Topic: %s, BrokersSent: %s", // + times, // + (System.currentTimeMillis() - beginTimestamp), // + msg.getTopic(),// + Arrays.toString(brokersSent)); + + throw new MQClientException(info, exception); + } + + List nsList = this.getmQClientFactory().getMQClientAPIImpl().getNameServerAddressList(); + if (null == nsList || nsList.isEmpty()) { + // 说明没有设置Name Server地址 + throw new MQClientException("No name server address, please set it." + + FAQUrl.suggestTodo(FAQUrl.NAME_SERVER_ADDR_NOT_EXIST_URL), null); + } + + throw new MQClientException("No route info of this topic, " + msg.getTopic() + + FAQUrl.suggestTodo(FAQUrl.NO_TOPIC_ROUTE_INFO), null); + } + + + /** + * 尝试寻找Topic路由信息，如果没有则到Name Server上找，再没有，则取默认Topic + */ + private TopicPublishInfo tryToFindTopicPublishInfo(final String topic) { + TopicPublishInfo topicPublishInfo = this.topicPublishInfoTable.get(topic); + if (null == topicPublishInfo || !topicPublishInfo.ok()) { + this.topicPublishInfoTable.putIfAbsent(topic, new TopicPublishInfo()); + this.mQClientFactory.updateTopicRouteInfoFromNameServer(topic); + topicPublishInfo = this.topicPublishInfoTable.get(topic); + } + + if (topicPublishInfo.isHaveTopicRouterInfo() || (topicPublishInfo != null && topicPublishInfo.ok())) { + return topicPublishInfo; + } + else { + this.mQClientFactory.updateTopicRouteInfoFromNameServer(topic, true, this.defaultMQProducer); + topicPublishInfo = this.topicPublishInfoTable.get(topic); + return topicPublishInfo; + } + } + + + private SendResult sendKernelImpl(final Message msg,// + final MessageQueue mq,// + final CommunicationMode communicationMode,// + final SendCallback sendCallback// + ) throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + String brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(mq.getBrokerName()); + if (null == brokerAddr) { + // TODO 此处可能对Name Server压力过大，需要调优 + tryToFindTopicPublishInfo(mq.getTopic()); + brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(mq.getBrokerName()); + } + + SendMessageContext context = null; + if (brokerAddr != null) { + byte[] prevBody = msg.getBody(); + try { + int sysFlag = 0; + if (this.tryToCompressMessage(msg)) { + sysFlag |= MessageSysFlag.CompressedFlag; + } + + final String tranMsg = msg.getProperty(MessageConst.PROPERTY_TRANSACTION_PREPARED); + if (tranMsg != null && Boolean.parseBoolean(tranMsg)) { + sysFlag |= MessageSysFlag.TransactionPreparedType; + } + + // 发消息之前，读写权限控制时调用 Hook + if (hasCheckForbiddenHook()) { + CheckForbiddenContext checkForbiddenContext = new CheckForbiddenContext(); + checkForbiddenContext.setNameSrvAddr(this.defaultMQProducer.getNamesrvAddr()); + checkForbiddenContext.setGroup(this.defaultMQProducer.getProducerGroup()); + checkForbiddenContext.setCommunicationMode(communicationMode); + checkForbiddenContext.setBrokerAddr(brokerAddr); + checkForbiddenContext.setMessage(msg); + checkForbiddenContext.setMq(mq); + checkForbiddenContext.setUnitMode(this.isUnitMode()); + this.executeCheckForbiddenHook(checkForbiddenContext); + } + + // 执行hook + if (this.hasSendMessageHook()) { + context = new SendMessageContext(); + context.setProducerGroup(this.defaultMQProducer.getProducerGroup()); + context.setCommunicationMode(communicationMode); + context.setBornHost(this.defaultMQProducer.getClientIP()); + context.setBrokerAddr(brokerAddr); + context.setMessage(msg); + context.setMq(mq); + this.executeSendMessageHookBefore(context); + } + + SendMessageRequestHeader requestHeader = new SendMessageRequestHeader(); + requestHeader.setProducerGroup(this.defaultMQProducer.getProducerGroup()); + requestHeader.setTopic(msg.getTopic()); + requestHeader.setDefaultTopic(this.defaultMQProducer.getCreateTopicKey()); + requestHeader.setDefaultTopicQueueNums(this.defaultMQProducer.getDefaultTopicQueueNums()); + requestHeader.setQueueId(mq.getQueueId()); + requestHeader.setSysFlag(sysFlag); + requestHeader.setBornTimestamp(System.currentTimeMillis()); + requestHeader.setFlag(msg.getFlag()); + requestHeader.setProperties(MessageDecoder.messageProperties2String(msg.getProperties())); + requestHeader.setReconsumeTimes(0); + requestHeader.setUnitMode(this.isUnitMode()); + if (requestHeader.getTopic().startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) { + String reconsumeTimes = MessageAccessor.getReconsumeTime(msg); + if (reconsumeTimes != null) { + requestHeader.setReconsumeTimes(new Integer(reconsumeTimes)); + MessageAccessor.clearProperty(msg, MessageConst.PROPERTY_RECONSUME_TIME); + } + } + + SendResult sendResult = this.mQClientFactory.getMQClientAPIImpl().sendMessage(// + brokerAddr,// 1 + mq.getBrokerName(),// 2 + msg,// 3 + requestHeader,// 4 + this.defaultMQProducer.getSendMsgTimeout(),// 5 + communicationMode,// 6 + sendCallback// 7 + ); + + // 执行hook + if (this.hasSendMessageHook()) { + context.setSendResult(sendResult); + this.executeSendMessageHookAfter(context); + } + + return sendResult; + } + catch (RemotingException e) { + // 执行hook + if (this.hasSendMessageHook()) { + context.setException(e); + this.executeSendMessageHookAfter(context); + } + throw e; + } + catch (MQBrokerException e) { + // 执行hook + if (this.hasSendMessageHook()) { + context.setException(e); + this.executeSendMessageHookAfter(context); + } + throw e; + } + catch (InterruptedException e) { + // 执行hook + if (this.hasSendMessageHook()) { + context.setException(e); + this.executeSendMessageHookAfter(context); + } + throw e; + } + finally { + msg.setBody(prevBody); + } + } + + throw new MQClientException("The broker[" + mq.getBrokerName() + "] not exist", null); + } + + /** + * 消息压缩level，默认5 + */ + private int zipCompressLevel = Integer.parseInt(System.getProperty(MixAll.MESSAGE_COMPRESS_LEVEL, "5")); + + + private boolean tryToCompressMessage(final Message msg) { + byte[] body = msg.getBody(); + if (body != null) { + if (body.length >= this.defaultMQProducer.getCompressMsgBodyOverHowmuch()) { + try { + byte[] data = UtilAll.compress(body, zipCompressLevel); + if (data != null) { + msg.setBody(data); + return true; + } + } + catch (IOException e) { + log.error("tryToCompressMessage exception", e); + log.warn(msg.toString()); + } + } + } + + return false; + } + + + /** + * DEFAULT ONEWAY ------------------------------------------------------- + */ + public void sendOneway(Message msg) throws MQClientException, RemotingException, InterruptedException { + try { + this.sendDefaultImpl(msg, CommunicationMode.ONEWAY, null); + } + catch (MQBrokerException e) { + throw new MQClientException("unknow exception", e); + } + } + + + /** + * KERNEL SYNC ------------------------------------------------------- + */ + public SendResult send(Message msg, MessageQueue mq) throws MQClientException, RemotingException, + MQBrokerException, InterruptedException { + // 有效性检查 + this.makeSureStateOK(); + Validators.checkMessage(msg, this.defaultMQProducer); + + if (!msg.getTopic().equals(mq.getTopic())) { + throw new MQClientException("message's topic not equal mq's topic", null); + } + + return this.sendKernelImpl(msg, mq, CommunicationMode.SYNC, null); + } + + + /** + * KERNEL ASYNC ------------------------------------------------------- + */ + public void send(Message msg, MessageQueue mq, SendCallback sendCallback) throws MQClientException, + RemotingException, InterruptedException { + // 有效性检查 + this.makeSureStateOK(); + Validators.checkMessage(msg, this.defaultMQProducer); + + if (!msg.getTopic().equals(mq.getTopic())) { + throw new MQClientException("message's topic not equal mq's topic", null); + } + + try { + this.sendKernelImpl(msg, mq, CommunicationMode.ASYNC, sendCallback); + } + catch (MQBrokerException e) { + throw new MQClientException("unknow exception", e); + } + } + + + /** + * KERNEL ONEWAY ------------------------------------------------------- + */ + public void sendOneway(Message msg, MessageQueue mq) throws MQClientException, RemotingException, + InterruptedException { + // 有效性检查 + this.makeSureStateOK(); + Validators.checkMessage(msg, this.defaultMQProducer); + + try { + this.sendKernelImpl(msg, mq, CommunicationMode.ONEWAY, null); + } + catch (MQBrokerException e) { + throw new MQClientException("unknow exception", e); + } + } + + + /** + * SELECT SYNC ------------------------------------------------------- + */ + public SendResult send(Message msg, MessageQueueSelector selector, Object arg) throws MQClientException, + RemotingException, MQBrokerException, InterruptedException { + return this.sendSelectImpl(msg, selector, arg, CommunicationMode.SYNC, null); + } + + + private SendResult sendSelectImpl(// + Message msg,// + MessageQueueSelector selector,// + Object arg,// + final CommunicationMode communicationMode,// + final SendCallback sendCallback// + ) throws MQClientException, RemotingException, MQBrokerException, InterruptedException { + // 有效性检查 + this.makeSureStateOK(); + Validators.checkMessage(msg, this.defaultMQProducer); + + TopicPublishInfo topicPublishInfo = this.tryToFindTopicPublishInfo(msg.getTopic()); + if (topicPublishInfo != null && topicPublishInfo.ok()) { + MessageQueue mq = null; + try { + mq = selector.select(topicPublishInfo.getMessageQueueList(), msg, arg); + } + catch (Throwable e) { + throw new MQClientException("select message queue throwed exception.", e); + } + + if (mq != null) { + return this.sendKernelImpl(msg, mq, communicationMode, sendCallback); + } + else { + throw new MQClientException("select message queue return null.", null); + } + } + + throw new MQClientException("No route info for this topic, " + msg.getTopic(), null); + } + + + /** + * SELECT ASYNC ------------------------------------------------------- + */ + public void send(Message msg, MessageQueueSelector selector, Object arg, SendCallback sendCallback) + throws MQClientException, RemotingException, InterruptedException { + try { + this.sendSelectImpl(msg, selector, arg, CommunicationMode.ASYNC, sendCallback); + } + catch (MQBrokerException e) { + throw new MQClientException("unknow exception", e); + } + } + + + /** + * SELECT ONEWAY ------------------------------------------------------- + */ + public void sendOneway(Message msg, MessageQueueSelector selector, Object arg) throws MQClientException, + RemotingException, InterruptedException { + try { + this.sendSelectImpl(msg, selector, arg, CommunicationMode.ONEWAY, null); + } + catch (MQBrokerException e) { + throw new MQClientException("unknow exception", e); + } + } + + + public TransactionSendResult sendMessageInTransaction(final Message msg, + final LocalTransactionExecuter tranExecuter, final Object arg) throws MQClientException { + // 有效性检查 + if (null == tranExecuter) { + throw new MQClientException("tranExecutor is null", null); + } + Validators.checkMessage(msg, this.defaultMQProducer); + + // 第一步，向Broker发送一条Prepared消息 + SendResult sendResult = null; + MessageAccessor.putProperty(msg, MessageConst.PROPERTY_TRANSACTION_PREPARED, "true"); + MessageAccessor.putProperty(msg, MessageConst.PROPERTY_PRODUCER_GROUP, + this.defaultMQProducer.getProducerGroup()); + try { + sendResult = this.send(msg); + } + catch (Exception e) { + throw new MQClientException("send message Exception", e); + } + + // 第二步，回调本地事务 + LocalTransactionState localTransactionState = LocalTransactionState.UNKNOW; + Throwable localException = null; + switch (sendResult.getSendStatus()) { + case SEND_OK: { + try { + localTransactionState = tranExecuter.executeLocalTransactionBranch(msg, arg); + if (null == localTransactionState) { + localTransactionState = LocalTransactionState.UNKNOW; + } + + if (localTransactionState != LocalTransactionState.COMMIT_MESSAGE) { + log.info("executeLocalTransactionBranch return {}", localTransactionState); + log.info(msg.toString()); + } + } + catch (Throwable e) { + log.info("executeLocalTransactionBranch exception", e); + log.info(msg.toString()); + localException = e; + } + } + break; + case FLUSH_DISK_TIMEOUT: + case FLUSH_SLAVE_TIMEOUT: + case SLAVE_NOT_AVAILABLE: + localTransactionState = LocalTransactionState.ROLLBACK_MESSAGE; + break; + default: + break; + } + + // 第三步，提交或者回滚Broker端消息 + try { + this.endTransaction(sendResult, localTransactionState, localException); + } + catch (Exception e) { + log.warn("local transaction execute " + localTransactionState + + ", but end broker transaction failed", e); + } + + TransactionSendResult transactionSendResult = new TransactionSendResult(); + transactionSendResult.setSendStatus(sendResult.getSendStatus()); + transactionSendResult.setMessageQueue(sendResult.getMessageQueue()); + transactionSendResult.setMsgId(sendResult.getMsgId()); + transactionSendResult.setQueueOffset(sendResult.getQueueOffset()); + transactionSendResult.setLocalTransactionState(localTransactionState); + return transactionSendResult; + } + + + private void endTransaction(// + final SendResult sendResult, // + final LocalTransactionState localTransactionState, // + final Throwable localException) throws RemotingException, MQBrokerException, + InterruptedException, UnknownHostException { + final MessageId id = MessageDecoder.decodeMessageId(sendResult.getMsgId()); + final String addr = RemotingUtil.socketAddress2String(id.getAddress()); + EndTransactionRequestHeader requestHeader = new EndTransactionRequestHeader(); + requestHeader.setCommitLogOffset(id.getOffset()); + switch (localTransactionState) { + case COMMIT_MESSAGE: + requestHeader.setCommitOrRollback(MessageSysFlag.TransactionCommitType); + break; + case ROLLBACK_MESSAGE: + requestHeader.setCommitOrRollback(MessageSysFlag.TransactionRollbackType); + break; + case UNKNOW: + requestHeader.setCommitOrRollback(MessageSysFlag.TransactionNotType); + break; + default: + break; + } + + requestHeader.setProducerGroup(this.defaultMQProducer.getProducerGroup()); + requestHeader.setTranStateTableOffset(sendResult.getQueueOffset()); + requestHeader.setMsgId(sendResult.getMsgId()); + String remark = + localException != null ? ("executeLocalTransactionBranch exception: " + localException + .toString()) : null; + this.mQClientFactory.getMQClientAPIImpl().endTransactionOneway(addr, requestHeader, remark, + this.defaultMQProducer.getSendMsgTimeout()); + } + + + /** + * DEFAULT SYNC ------------------------------------------------------- + */ + public SendResult send(Message msg) throws MQClientException, RemotingException, MQBrokerException, + InterruptedException { + return this.sendDefaultImpl(msg, CommunicationMode.SYNC, null); + } + + + public ConcurrentHashMap getTopicPublishInfoTable() { + return topicPublishInfoTable; + } + + + public MQClientInstance getmQClientFactory() { + return mQClientFactory; + } + + + public int getZipCompressLevel() { + return zipCompressLevel; + } + + + public void setZipCompressLevel(int zipCompressLevel) { + this.zipCompressLevel = zipCompressLevel; + } + + + public ServiceState getServiceState() { + return serviceState; + } + + + public void setServiceState(ServiceState serviceState) { + this.serviceState = serviceState; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/producer/MQProducerInner.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/producer/MQProducerInner.java new file mode 100644 index 000000000..724600857 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/producer/MQProducerInner.java @@ -0,0 +1,51 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.impl.producer; + +import java.util.Set; + +import com.alibaba.rocketmq.client.producer.TransactionCheckListener; +import com.alibaba.rocketmq.common.message.MessageExt; +import com.alibaba.rocketmq.common.protocol.header.CheckTransactionStateRequestHeader; + + +/** + * Producer内部接口 + * + * @author shijia.wxr + * @since 2013-7-24 + */ +public interface MQProducerInner { + public Set getPublishTopicList(); + + + public boolean isPublishTopicNeedUpdate(final String topic); + + + public TransactionCheckListener checkListener(); + + + public void checkTransactionState(// + final String addr, // + final MessageExt msg, // + final CheckTransactionStateRequestHeader checkRequestHeader); + + + public void updateTopicPublishInfo(final String topic, final TopicPublishInfo info); + + + public boolean isUnitMode(); +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/producer/TopicPublishInfo.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/producer/TopicPublishInfo.java new file mode 100644 index 000000000..46fbc62e4 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/impl/producer/TopicPublishInfo.java @@ -0,0 +1,112 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.impl.producer; + +import com.alibaba.rocketmq.common.message.MessageQueue; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + + +/** + * 发布Topic用到的路由信息 + * + * @author shijia.wxr + * @since 2013-7-24 + */ +public class TopicPublishInfo { + private boolean orderTopic = false; + private boolean haveTopicRouterInfo = false; + private List messageQueueList = new ArrayList(); + private AtomicInteger sendWhichQueue = new AtomicInteger(0); + + + public boolean isOrderTopic() { + return orderTopic; + } + + + public boolean ok() { + return null != this.messageQueueList && !this.messageQueueList.isEmpty(); + } + + + public void setOrderTopic(boolean orderTopic) { + this.orderTopic = orderTopic; + } + + + public List getMessageQueueList() { + return messageQueueList; + } + + + public void setMessageQueueList(List messageQueueList) { + this.messageQueueList = messageQueueList; + } + + + public AtomicInteger getSendWhichQueue() { + return sendWhichQueue; + } + + + public void setSendWhichQueue(AtomicInteger sendWhichQueue) { + this.sendWhichQueue = sendWhichQueue; + } + + + public boolean isHaveTopicRouterInfo() { + return haveTopicRouterInfo; + } + + + public void setHaveTopicRouterInfo(boolean haveTopicRouterInfo) { + this.haveTopicRouterInfo = haveTopicRouterInfo; + } + + + /** + * 如果lastBrokerName不为null，则寻找与其不同的MessageQueue + */ + public MessageQueue selectOneMessageQueue(final String lastBrokerName) { + if (lastBrokerName != null) { + int index = this.sendWhichQueue.getAndIncrement(); + for (int i = 0; i < this.messageQueueList.size(); i++) { + int pos = Math.abs(index++) % this.messageQueueList.size(); + MessageQueue mq = this.messageQueueList.get(pos); + if (!mq.getBrokerName().equals(lastBrokerName)) { + return mq; + } + } + + return null; + } + else { + int index = this.sendWhichQueue.getAndIncrement(); + int pos = Math.abs(index) % this.messageQueueList.size(); + return this.messageQueueList.get(pos); + } + } + + + @Override + public String toString() { + return "TopicPublishInfo [orderTopic=" + orderTopic + ", messageQueueList=" + messageQueueList + + ", sendWhichQueue=" + sendWhichQueue + ", haveTopicRouterInfo=" + haveTopicRouterInfo + "]"; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/log/ClientLogger.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/log/ClientLogger.java new file mode 100644 index 000000000..e78bfae7d --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/log/ClientLogger.java @@ -0,0 +1,117 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.log; + +import java.lang.reflect.Method; +import java.net.URL; + +import org.slf4j.ILoggerFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.rocketmq.common.constant.LoggerName; + + +/** + * Client通过反射来初始化客户端日志 + * + * @author 菱叶 + * @since 2013-7-24 + */ +public class ClientLogger { + private static Logger log; + + static { + // 初始化Logger + log = createLogger(LoggerName.ClientLoggerName); + } + + + private static Logger createLogger(final String loggerName) { + String logConfigFilePath = + System.getProperty("rocketmq.client.log.configFile", + System.getenv("ROCKETMQ_CLIENT_LOG_CONFIGFILE")); + Boolean isloadconfig = + Boolean.parseBoolean(System.getProperty("rocketmq.client.log.loadconfig", "true")); + + final String log4j_resource_file = + System.getProperty("rocketmq.client.log4j.resource.fileName", "log4j_rocketmq_client.xml"); + + final String logback_resource_file = + System + .getProperty("rocketmq.client.logback.resource.fileName", "logback_rocketmq_client.xml"); + + if (isloadconfig) { + try { + ILoggerFactory iLoggerFactory = LoggerFactory.getILoggerFactory(); + Class classType = iLoggerFactory.getClass(); + if (classType.getName().equals("org.slf4j.impl.Log4jLoggerFactory")) { + Class DOMConfigurator = null; + Object DOMConfiguratorObj = null; + DOMConfigurator = Class.forName("org.apache.log4j.xml.DOMConfigurator"); + DOMConfiguratorObj = DOMConfigurator.newInstance(); + if (null == logConfigFilePath) { + // 如果应用没有配置，则使用jar包内置配置 + Method configure = DOMConfiguratorObj.getClass().getMethod("configure", URL.class); + URL url = ClientLogger.class.getClassLoader().getResource(log4j_resource_file); + configure.invoke(DOMConfiguratorObj, url); + } + else { + Method configure = DOMConfiguratorObj.getClass().getMethod("configure", String.class); + configure.invoke(DOMConfiguratorObj, logConfigFilePath); + } + + } + else if (classType.getName().equals("ch.qos.logback.classic.LoggerContext")) { + Class joranConfigurator = null; + Class context = Class.forName("ch.qos.logback.core.Context"); + Object joranConfiguratoroObj = null; + joranConfigurator = Class.forName("ch.qos.logback.classic.joran.JoranConfigurator"); + joranConfiguratoroObj = joranConfigurator.newInstance(); + Method setContext = joranConfiguratoroObj.getClass().getMethod("setContext", context); + setContext.invoke(joranConfiguratoroObj, iLoggerFactory); + if (null == logConfigFilePath) { + // 如果应用没有配置，则使用jar包内置配置 + URL url = ClientLogger.class.getClassLoader().getResource(logback_resource_file); + Method doConfigure = + joranConfiguratoroObj.getClass().getMethod("doConfigure", URL.class); + doConfigure.invoke(joranConfiguratoroObj, url); + } + else { + Method doConfigure = + joranConfiguratoroObj.getClass().getMethod("doConfigure", String.class); + doConfigure.invoke(joranConfiguratoroObj, logConfigFilePath); + } + + } + } + catch (Exception e) { + System.err.println(e); + } + } + return LoggerFactory.getLogger(LoggerName.ClientLoggerName); + } + + + public static Logger getLog() { + return log; + } + + + public static void setLog(Logger log) { + ClientLogger.log = log; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/DefaultMQProducer.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/DefaultMQProducer.java new file mode 100644 index 000000000..2e83799ab --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/DefaultMQProducer.java @@ -0,0 +1,333 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.producer; + +import java.util.List; + +import com.alibaba.rocketmq.client.ClientConfig; +import com.alibaba.rocketmq.client.QueryResult; +import com.alibaba.rocketmq.client.exception.MQBrokerException; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.client.impl.producer.DefaultMQProducerImpl; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.message.Message; +import com.alibaba.rocketmq.common.message.MessageExt; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.remoting.exception.RemotingException; + + +/** + * 消息生产者，适合使用spring初始化 + * + * @author shijia.wxr + * @since 2013-7-25 + */ +public class DefaultMQProducer extends ClientConfig implements MQProducer { + protected final transient DefaultMQProducerImpl defaultMQProducerImpl; + /** + * 一般发送同样消息的Producer，归为同一个Group，应用必须设置，并保证命名唯一 + */ + private String producerGroup; + /** + * 支持在发送消息时，如果Topic不存在，自动创建Topic，但是要指定Key + */ + private String createTopicKey = MixAll.DEFAULT_TOPIC; + /** + * 发送消息，自动创建Topic时，默认队列数 + */ + private volatile int defaultTopicQueueNums = 4; + /** + * 发送消息超时，不建议修改 + */ + private int sendMsgTimeout = 3000; + /** + * Message Body大小超过阀值，则压缩 + */ + private int compressMsgBodyOverHowmuch = 1024 * 4; + /** + * 发送失败后，重试几次 + */ + private int retryTimesWhenSendFailed = 2; + /** + * 消息已经成功写入Master，但是刷盘超时或者同步到Slave失败，则尝试重试另一个Broker，不建议修改默认值
+ * 顺序消息无效 + */ + private boolean retryAnotherBrokerWhenNotStoreOK = false; + /** + * 最大消息大小，默认512K + */ + private int maxMessageSize = 1024 * 128; + /** + * 是否为单元化的发布者 + */ + private boolean unitMode = false; + + + public DefaultMQProducer() { + this(MixAll.DEFAULT_PRODUCER_GROUP, null); + } + + + public DefaultMQProducer(final String producerGroup) { + this(producerGroup, null); + } + + + public DefaultMQProducer(RPCHook rpcHook) { + this(MixAll.DEFAULT_PRODUCER_GROUP, rpcHook); + } + + + public DefaultMQProducer(final String producerGroup, RPCHook rpcHook) { + this.producerGroup = producerGroup; + defaultMQProducerImpl = new DefaultMQProducerImpl(this, rpcHook); + } + + + @Override + public void start() throws MQClientException { + this.defaultMQProducerImpl.start(); + } + + + @Override + public void shutdown() { + this.defaultMQProducerImpl.shutdown(); + } + + + @Override + public List fetchPublishMessageQueues(String topic) throws MQClientException { + return this.defaultMQProducerImpl.fetchPublishMessageQueues(topic); + } + + + @Override + public SendResult send(Message msg) throws MQClientException, RemotingException, MQBrokerException, + InterruptedException { + return this.defaultMQProducerImpl.send(msg); + } + + + @Override + public void send(Message msg, SendCallback sendCallback) throws MQClientException, RemotingException, + InterruptedException { + this.defaultMQProducerImpl.send(msg, sendCallback); + } + + + @Override + public void sendOneway(Message msg) throws MQClientException, RemotingException, InterruptedException { + this.defaultMQProducerImpl.sendOneway(msg); + } + + + @Override + public SendResult send(Message msg, MessageQueue mq) throws MQClientException, RemotingException, + MQBrokerException, InterruptedException { + return this.defaultMQProducerImpl.send(msg, mq); + } + + + @Override + public void send(Message msg, MessageQueue mq, SendCallback sendCallback) throws MQClientException, + RemotingException, InterruptedException { + this.defaultMQProducerImpl.send(msg, mq, sendCallback); + } + + + @Override + public void sendOneway(Message msg, MessageQueue mq) throws MQClientException, RemotingException, + InterruptedException { + this.defaultMQProducerImpl.sendOneway(msg, mq); + } + + + @Override + public SendResult send(Message msg, MessageQueueSelector selector, Object arg) throws MQClientException, + RemotingException, MQBrokerException, InterruptedException { + return this.defaultMQProducerImpl.send(msg, selector, arg); + } + + + @Override + public void send(Message msg, MessageQueueSelector selector, Object arg, SendCallback sendCallback) + throws MQClientException, RemotingException, InterruptedException { + this.defaultMQProducerImpl.send(msg, selector, arg, sendCallback); + } + + + @Override + public void sendOneway(Message msg, MessageQueueSelector selector, Object arg) throws MQClientException, + RemotingException, InterruptedException { + this.defaultMQProducerImpl.sendOneway(msg, selector, arg); + } + + + @Override + public TransactionSendResult sendMessageInTransaction(Message msg, LocalTransactionExecuter tranExecuter, + final Object arg) throws MQClientException { + throw new RuntimeException( + "sendMessageInTransaction not implement, please use TransactionMQProducer class"); + } + + + @Override + public void createTopic(String key, String newTopic, int queueNum) throws MQClientException { + createTopic(key, newTopic, queueNum, 0); + } + + + @Override + public void createTopic(String key, String newTopic, int queueNum, int topicSysFlag) + throws MQClientException { + this.defaultMQProducerImpl.createTopic(key, newTopic, queueNum, topicSysFlag); + } + + + @Override + public long searchOffset(MessageQueue mq, long timestamp) throws MQClientException { + return this.defaultMQProducerImpl.searchOffset(mq, timestamp); + } + + + @Override + public long maxOffset(MessageQueue mq) throws MQClientException { + return this.defaultMQProducerImpl.maxOffset(mq); + } + + + @Override + public long minOffset(MessageQueue mq) throws MQClientException { + return this.defaultMQProducerImpl.minOffset(mq); + } + + + @Override + public long earliestMsgStoreTime(MessageQueue mq) throws MQClientException { + return this.defaultMQProducerImpl.earliestMsgStoreTime(mq); + } + + + @Override + public MessageExt viewMessage(String msgId) throws RemotingException, MQBrokerException, + InterruptedException, MQClientException { + return this.defaultMQProducerImpl.viewMessage(msgId); + } + + + @Override + public QueryResult queryMessage(String topic, String key, int maxNum, long begin, long end) + throws MQClientException, InterruptedException { + return this.defaultMQProducerImpl.queryMessage(topic, key, maxNum, begin, end); + } + + + public String getProducerGroup() { + return producerGroup; + } + + + public void setProducerGroup(String producerGroup) { + this.producerGroup = producerGroup; + } + + + public String getCreateTopicKey() { + return createTopicKey; + } + + + public void setCreateTopicKey(String createTopicKey) { + this.createTopicKey = createTopicKey; + } + + + public int getSendMsgTimeout() { + return sendMsgTimeout; + } + + + public void setSendMsgTimeout(int sendMsgTimeout) { + this.sendMsgTimeout = sendMsgTimeout; + } + + + public int getCompressMsgBodyOverHowmuch() { + return compressMsgBodyOverHowmuch; + } + + + public void setCompressMsgBodyOverHowmuch(int compressMsgBodyOverHowmuch) { + this.compressMsgBodyOverHowmuch = compressMsgBodyOverHowmuch; + } + + + public DefaultMQProducerImpl getDefaultMQProducerImpl() { + return defaultMQProducerImpl; + } + + + public boolean isRetryAnotherBrokerWhenNotStoreOK() { + return retryAnotherBrokerWhenNotStoreOK; + } + + + public void setRetryAnotherBrokerWhenNotStoreOK(boolean retryAnotherBrokerWhenNotStoreOK) { + this.retryAnotherBrokerWhenNotStoreOK = retryAnotherBrokerWhenNotStoreOK; + } + + + public int getMaxMessageSize() { + return maxMessageSize; + } + + + public void setMaxMessageSize(int maxMessageSize) { + this.maxMessageSize = maxMessageSize; + } + + + public int getDefaultTopicQueueNums() { + return defaultTopicQueueNums; + } + + + public void setDefaultTopicQueueNums(int defaultTopicQueueNums) { + this.defaultTopicQueueNums = defaultTopicQueueNums; + } + + + public int getRetryTimesWhenSendFailed() { + return retryTimesWhenSendFailed; + } + + + public void setRetryTimesWhenSendFailed(int retryTimesWhenSendFailed) { + this.retryTimesWhenSendFailed = retryTimesWhenSendFailed; + } + + + public boolean isUnitMode() { + return unitMode; + } + + + public void setUnitMode(boolean isUnitMode) { + this.unitMode = isUnitMode; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/LocalTransactionExecuter.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/LocalTransactionExecuter.java new file mode 100644 index 000000000..28017eee9 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/LocalTransactionExecuter.java @@ -0,0 +1,29 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.producer; + +import com.alibaba.rocketmq.common.message.Message; + + +/** + * 执行本地事务，由客户端回调 + * + * @author shijia.wxr + * @since 2013-7-25 + */ +public interface LocalTransactionExecuter { + public LocalTransactionState executeLocalTransactionBranch(final Message msg, final Object arg); +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/LocalTransactionState.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/LocalTransactionState.java new file mode 100644 index 000000000..5894bbebb --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/LocalTransactionState.java @@ -0,0 +1,30 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.producer; + +/** + * Producer本地事务执行状态 + * + * @author shijia.wxr + * @since 2013-7-25 + */ +public enum LocalTransactionState { + // 提交事务 + COMMIT_MESSAGE, + // 回滚事务 + ROLLBACK_MESSAGE, + UNKNOW, +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/MQProducer.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/MQProducer.java new file mode 100644 index 000000000..b744c6e41 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/MQProducer.java @@ -0,0 +1,216 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.producer; + +import java.util.List; + +import com.alibaba.rocketmq.client.MQAdmin; +import com.alibaba.rocketmq.client.exception.MQBrokerException; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.common.message.Message; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.remoting.exception.RemotingException; + + +/** + * 消息生产者 + * + * @author shijia.wxr + * @since 2013-7-25 + */ +public interface MQProducer extends MQAdmin { + /** + * 启动服务 + * + * @throws MQClientException + */ + public void start() throws MQClientException; + + + /** + * 关闭服务，一旦关闭，此对象将不可用 + */ + public void shutdown(); + + + /** + * 根据topic获取对应的MessageQueue，如果是顺序消息，则按照顺序消息配置返回 + * + * @param topic + * 消息Topic + * @return 返回队列集合 + * @throws MQClientException + */ + public List fetchPublishMessageQueues(final String topic) throws MQClientException; + + + /** + * 发送消息，同步调用 + * + * @param msg + * 消息 + * @return 发送结果 + * @throws InterruptedException + * @throws MQBrokerException + * @throws RemotingException + * @throws MQClientException + */ + public SendResult send(final Message msg) throws MQClientException, RemotingException, MQBrokerException, + InterruptedException; + + + /** + * 发送消息，异步调用 + * + * @param msg + * 消息 + * @param sendCallback + * 发送结果通过此接口回调 + * @throws MQClientException + * @throws RemotingException + * @throws InterruptedException + */ + public void send(final Message msg, final SendCallback sendCallback) throws MQClientException, + RemotingException, InterruptedException; + + + /** + * 发送消息，Oneway形式，服务器不应答，无法保证消息是否成功到达服务器 + * + * @param msg + * 消息 + * @throws MQClientException + * @throws RemotingException + * @throws InterruptedException + */ + public void sendOneway(final Message msg) throws MQClientException, RemotingException, + InterruptedException; + + + /** + * 向指定队列发送消息，同步调用 + * + * @param msg + * 消息 + * @param mq + * 队列 + * @return 发送结果 + * @throws InterruptedException + * @throws MQBrokerException + * @throws RemotingException + * @throws MQClientException + */ + public SendResult send(final Message msg, final MessageQueue mq) throws MQClientException, + RemotingException, MQBrokerException, InterruptedException; + + + /** + * 向指定队列发送消息，异步调用 + * + * @param msg + * 消息 + * @param mq + * 队列 + * @param sendCallback + * 发送结果通过此接口回调 + * @throws InterruptedException + * @throws RemotingException + * @throws MQClientException + */ + public void send(final Message msg, final MessageQueue mq, final SendCallback sendCallback) + throws MQClientException, RemotingException, InterruptedException; + + + /** + * 向指定队列发送消息，Oneway形式，服务器不应答，无法保证消息是否成功到达服务器 + * + * @param msg + * 消息 + * @param mq + * 队列 + * @throws MQClientException + * @throws RemotingException + * @throws InterruptedException + */ + public void sendOneway(final Message msg, final MessageQueue mq) throws MQClientException, + RemotingException, InterruptedException; + + + /** + * 发送消息，可以自定义选择队列，队列的总数可能会由于Broker的启停变化
+ * 如果要保证消息严格有序，在向运维人员申请Topic时，需要特别说明
+ * 同步调用 + * + * @param msg + * 消息 + * @param selector + * 队列选择器，发送时会回调 + * @param arg + * 回调队列选择器时，此参数会传入队列选择方法 + * @return 发送结果 + * @throws InterruptedException + * @throws MQBrokerException + * @throws RemotingException + * @throws MQClientException + */ + public SendResult send(final Message msg, final MessageQueueSelector selector, final Object arg) + throws MQClientException, RemotingException, MQBrokerException, InterruptedException; + + + /** + * 发送消息，可以自定义选择队列，队列的总数可能会由于Broker的启停变化
+ * 如果要保证消息严格有序，在向运维人员申请Topic时，需要特别说明
+ * 异步调用 + * + * @param msg + * 消息 + * @param selector + * 队列选择器，发送时会回调 + * @param arg + * 回调队列选择器时，此参数会传入队列选择方法 + * @param sendCallback + * 发送结果通过此接口回调 + * @throws MQClientException + * @throws RemotingException + * @throws InterruptedException + */ + public void send(final Message msg, final MessageQueueSelector selector, final Object arg, + final SendCallback sendCallback) throws MQClientException, RemotingException, + InterruptedException; + + + /** + * 发送消息，可以自定义选择队列，队列的总数可能会由于Broker的启停变化
+ * 如果要保证消息严格有序，在向运维人员申请Topic时，需要特别说明
+ * Oneway形式，服务器不应答，无法保证消息是否成功到达服务器 + * + * @param msg + * 消息 + * @param selector + * 队列选择器，发送时会回调 + * @param arg + * 回调队列选择器时，此参数会传入队列选择方法 + * @throws MQClientException + * @throws RemotingException + * @throws InterruptedException + */ + public void sendOneway(final Message msg, final MessageQueueSelector selector, final Object arg) + throws MQClientException, RemotingException, InterruptedException; + + + public TransactionSendResult sendMessageInTransaction(final Message msg, + final LocalTransactionExecuter tranExecuter, final Object arg) throws MQClientException; +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/MessageQueueSelector.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/MessageQueueSelector.java new file mode 100644 index 000000000..71c67d3e1 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/MessageQueueSelector.java @@ -0,0 +1,32 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.producer; + +import java.util.List; + +import com.alibaba.rocketmq.common.message.Message; +import com.alibaba.rocketmq.common.message.MessageQueue; + + +/** + * 队列选择器 + * + * @author shijia.wxr + * @since 2013-7-25 + */ +public interface MessageQueueSelector { + public MessageQueue select(final List mqs, final Message msg, final Object arg); +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/SendCallback.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/SendCallback.java new file mode 100644 index 000000000..d2d946945 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/SendCallback.java @@ -0,0 +1,29 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.producer; + +/** + * 异步发送消息回调接口 + * + * @author shijia.wxr + * @since 2013-7-25 + */ +public interface SendCallback { + public void onSuccess(final SendResult sendResult); + + + public void onException(final Throwable e); +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/SendResult.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/SendResult.java new file mode 100644 index 000000000..fe8ac0f96 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/SendResult.java @@ -0,0 +1,99 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.producer; + +import com.alibaba.rocketmq.client.VirtualEnvUtil; +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.message.MessageQueue; + + +/** + * 发送消息结果 + * + * @author shijia.wxr + * @since 2013-7-25 + */ +public class SendResult { + private SendStatus sendStatus; + private String msgId; + private MessageQueue messageQueue; + private long queueOffset; + + + public SendResult() { + } + + + public SendResult(SendStatus sendStatus, String msgId, MessageQueue messageQueue, long queueOffset, + String projectGroupPrefix) { + this.sendStatus = sendStatus; + this.msgId = msgId; + this.messageQueue = messageQueue; + this.queueOffset = queueOffset; + // 清除虚拟运行环境相关的projectGroupPrefix + if (!UtilAll.isBlank(projectGroupPrefix)) { + this.messageQueue.setTopic(VirtualEnvUtil.clearProjectGroup(this.messageQueue.getTopic(), + projectGroupPrefix)); + } + } + + + public String getMsgId() { + return msgId; + } + + + public void setMsgId(String msgId) { + this.msgId = msgId; + } + + + public SendStatus getSendStatus() { + return sendStatus; + } + + + public void setSendStatus(SendStatus sendStatus) { + this.sendStatus = sendStatus; + } + + + public MessageQueue getMessageQueue() { + return messageQueue; + } + + + public void setMessageQueue(MessageQueue messageQueue) { + this.messageQueue = messageQueue; + } + + + public long getQueueOffset() { + return queueOffset; + } + + + public void setQueueOffset(long queueOffset) { + this.queueOffset = queueOffset; + } + + + @Override + public String toString() { + return "SendResult [sendStatus=" + sendStatus + ", msgId=" + msgId + ", messageQueue=" + messageQueue + + ", queueOffset=" + queueOffset + "]"; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/SendStatus.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/SendStatus.java new file mode 100644 index 000000000..ad7dac025 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/SendStatus.java @@ -0,0 +1,33 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.producer; + +/** + * 这4种状态都表示消息已经成功到达Master + * + * @author shijia.wxr + * @since 2013-7-25 + */ +public enum SendStatus { + // 消息发送成功 + SEND_OK, + // 消息发送成功，但是服务器刷盘超时，消息已经进入服务器队列，只有此时服务器宕机，消息才会丢失 + FLUSH_DISK_TIMEOUT, + // 消息发送成功，但是服务器同步到Slave时超时，消息已经进入服务器队列，只有此时服务器宕机，消息才会丢失 + FLUSH_SLAVE_TIMEOUT, + // 消息发送成功，但是此时slave不可用，消息已经进入服务器队列，只有此时服务器宕机，消息才会丢失 + SLAVE_NOT_AVAILABLE +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/TransactionCheckListener.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/TransactionCheckListener.java new file mode 100644 index 000000000..7a79f3f5f --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/TransactionCheckListener.java @@ -0,0 +1,29 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.producer; + +import com.alibaba.rocketmq.common.message.MessageExt; + + +/** + * 服务器回调Producer，检查本地事务分支成功还是失败 + * + * @author shijia.wxr + * @since 2013-7-25 + */ +public interface TransactionCheckListener { + public LocalTransactionState checkLocalTransactionState(final MessageExt msg); +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/TransactionMQProducer.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/TransactionMQProducer.java new file mode 100644 index 000000000..798b7f0d9 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/TransactionMQProducer.java @@ -0,0 +1,116 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.producer; + +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.common.message.Message; + + +/** + * 支持分布式事务Producer + * + * @author shijia.wxr + * @since 2013-7-25 + */ +public class TransactionMQProducer extends DefaultMQProducer { + private TransactionCheckListener transactionCheckListener; + /** + * 事务回查最小并发数 + */ + private int checkThreadPoolMinSize = 1; + /** + * 事务回查最大并发数 + */ + private int checkThreadPoolMaxSize = 1; + /** + * 事务回查队列数 + */ + private int checkRequestHoldMax = 2000; + + + public TransactionMQProducer() { + } + + + public TransactionMQProducer(final String producerGroup) { + super(producerGroup); + } + + + @Override + public void start() throws MQClientException { + this.defaultMQProducerImpl.initTransactionEnv(); + super.start(); + } + + + @Override + public void shutdown() { + super.shutdown(); + this.defaultMQProducerImpl.destroyTransactionEnv(); + } + + + @Override + public TransactionSendResult sendMessageInTransaction(final Message msg, + final LocalTransactionExecuter tranExecuter, final Object arg) throws MQClientException { + if (null == this.transactionCheckListener) { + throw new MQClientException("localTransactionBranchCheckListener is null", null); + } + + return this.defaultMQProducerImpl.sendMessageInTransaction(msg, tranExecuter, arg); + } + + + public TransactionCheckListener getTransactionCheckListener() { + return transactionCheckListener; + } + + + public void setTransactionCheckListener(TransactionCheckListener transactionCheckListener) { + this.transactionCheckListener = transactionCheckListener; + } + + + public int getCheckThreadPoolMinSize() { + return checkThreadPoolMinSize; + } + + + public void setCheckThreadPoolMinSize(int checkThreadPoolMinSize) { + this.checkThreadPoolMinSize = checkThreadPoolMinSize; + } + + + public int getCheckThreadPoolMaxSize() { + return checkThreadPoolMaxSize; + } + + + public void setCheckThreadPoolMaxSize(int checkThreadPoolMaxSize) { + this.checkThreadPoolMaxSize = checkThreadPoolMaxSize; + } + + + public int getCheckRequestHoldMax() { + return checkRequestHoldMax; + } + + + public void setCheckRequestHoldMax(int checkRequestHoldMax) { + this.checkRequestHoldMax = checkRequestHoldMax; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/TransactionSendResult.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/TransactionSendResult.java new file mode 100644 index 000000000..c2e101897 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/TransactionSendResult.java @@ -0,0 +1,40 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.producer; + +/** + * 发送事务消息返回结果 + * + * @author shijia.wxr + * @since 2013-7-31 + */ +public class TransactionSendResult extends SendResult { + private LocalTransactionState localTransactionState; + + + public TransactionSendResult() { + } + + + public LocalTransactionState getLocalTransactionState() { + return localTransactionState; + } + + + public void setLocalTransactionState(LocalTransactionState localTransactionState) { + this.localTransactionState = localTransactionState; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/selector/SelectMessageQueueByHash.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/selector/SelectMessageQueueByHash.java new file mode 100644 index 000000000..9d8ef368d --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/selector/SelectMessageQueueByHash.java @@ -0,0 +1,43 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.producer.selector; + +import java.util.List; + +import com.alibaba.rocketmq.client.producer.MessageQueueSelector; +import com.alibaba.rocketmq.common.message.Message; +import com.alibaba.rocketmq.common.message.MessageQueue; + + +/** + * 使用哈希算法来选择队列，顺序消息通常都这样做
+ * + * @author shijia.wxr + * @since 2013-6-27 + */ +public class SelectMessageQueueByHash implements MessageQueueSelector { + + @Override + public MessageQueue select(List mqs, Message msg, Object arg) { + int value = arg.hashCode(); + if (value < 0) { + value = Math.abs(value); + } + + value = value % mqs.size(); + return mqs.get(value); + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/selector/SelectMessageQueueByMachineRoom.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/selector/SelectMessageQueueByMachineRoom.java new file mode 100644 index 000000000..903cafb53 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/selector/SelectMessageQueueByMachineRoom.java @@ -0,0 +1,51 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.producer.selector; + +import java.util.List; +import java.util.Set; + +import com.alibaba.rocketmq.client.producer.MessageQueueSelector; +import com.alibaba.rocketmq.common.message.Message; +import com.alibaba.rocketmq.common.message.MessageQueue; + + +/** + * 根据机房来选择发往哪个队列，支付宝逻辑机房使用 + * + * @author shijia.wxr + * @since 2013-7-25 + */ +public class SelectMessageQueueByMachineRoom implements MessageQueueSelector { + private Set consumeridcs; + + + @Override + public MessageQueue select(List mqs, Message msg, Object arg) { + // TODO Auto-generated method stub + return null; + } + + + public Set getConsumeridcs() { + return consumeridcs; + } + + + public void setConsumeridcs(Set consumeridcs) { + this.consumeridcs = consumeridcs; + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/selector/SelectMessageQueueByRandoom.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/selector/SelectMessageQueueByRandoom.java new file mode 100644 index 000000000..e5d745aaf --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/producer/selector/SelectMessageQueueByRandoom.java @@ -0,0 +1,46 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.client.producer.selector; + +import java.util.List; +import java.util.Random; + +import com.alibaba.rocketmq.client.producer.MessageQueueSelector; +import com.alibaba.rocketmq.common.message.Message; +import com.alibaba.rocketmq.common.message.MessageQueue; + + +/** + * 发送消息，随机选择队列 + * + * @author shijia.wxr + * @since 2013-7-25 + */ +public class SelectMessageQueueByRandoom implements MessageQueueSelector { + private Random random = new Random(System.currentTimeMillis()); + + + @Override + public MessageQueue select(List mqs, Message msg, Object arg) { + int value = random.nextInt(); + if (value < 0) { + value = Math.abs(value); + } + + value = value % mqs.size(); + return mqs.get(value); + } +} diff --git a/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/stat/ConsumerStatsManager.java b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/stat/ConsumerStatsManager.java new file mode 100644 index 000000000..24eb2cb41 --- /dev/null +++ b/rocketmq-client/src/main/java/com/alibaba/rocketmq/client/stat/ConsumerStatsManager.java @@ -0,0 +1,155 @@ +package com.alibaba.rocketmq.client.stat; + +import java.util.concurrent.ScheduledExecutorService; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.common.protocol.body.ConsumeStatus; +import com.alibaba.rocketmq.common.stats.StatsItemSet; +import com.alibaba.rocketmq.common.stats.StatsSnapshot; + + +public class ConsumerStatsManager { + private static final Logger log = LoggerFactory.getLogger(LoggerName.ClientLoggerName); + + private static final String TOPIC_AND_GROUP_CONSUME_OK_TPS = "CONSUME_OK_TPS"; + private static final String TOPIC_AND_GROUP_CONSUME_FAILED_TPS = "CONSUME_FAILED_TPS"; + private static final String TOPIC_AND_GROUP_CONSUME_RT = "CONSUME_RT"; + private static final String TOPIC_AND_GROUP_PULL_TPS = "PULL_TPS"; + private static final String TOPIC_AND_GROUP_PULL_RT = "PULL_RT"; + + private final StatsItemSet topicAndGroupConsumeOKTPS; + private final StatsItemSet topicAndGroupConsumeRT; + private final StatsItemSet topicAndGroupConsumeFailedTPS; + private final StatsItemSet topicAndGroupPullTPS; + private final StatsItemSet topicAndGroupPullRT; + + + public ConsumerStatsManager(final ScheduledExecutorService scheduledExecutorService) { + this.topicAndGroupConsumeOKTPS = + new StatsItemSet(TOPIC_AND_GROUP_CONSUME_OK_TPS, scheduledExecutorService, log); + + this.topicAndGroupConsumeRT = + new StatsItemSet(TOPIC_AND_GROUP_CONSUME_RT, scheduledExecutorService, log); + + this.topicAndGroupConsumeFailedTPS = + new StatsItemSet(TOPIC_AND_GROUP_CONSUME_FAILED_TPS, scheduledExecutorService, log); + + this.topicAndGroupPullTPS = new StatsItemSet(TOPIC_AND_GROUP_PULL_TPS, scheduledExecutorService, log); + + this.topicAndGroupPullRT = new StatsItemSet(TOPIC_AND_GROUP_PULL_RT, scheduledExecutorService, log); + } + + + public void start() { + } + + + public void shutdown() { + } + + + public void incPullRT(final String group, final String topic, final long rt) { + this.topicAndGroupPullRT.addValue(topic + "@" + group, (int) rt, 1); + } + + + public void incPullTPS(final String group, final String topic, final long msgs) { + this.topicAndGroupPullTPS.addValue(topic + "@" + group, (int) msgs, 1); + } + + + public void incConsumeRT(final String group, final String topic, final long rt) { + this.topicAndGroupConsumeRT.addValue(topic + "@" + group, (int) rt, 1); + } + + + public void incConsumeOKTPS(final String group, final String topic, final long msgs) { + this.topicAndGroupConsumeOKTPS.addValue(topic + "@" + group, (int) msgs, 1); + } + + + public void incConsumeFailedTPS(final String group, final String topic, final long msgs) { + this.topicAndGroupConsumeFailedTPS.addValue(topic + "@" + group, (int) msgs, 1); + } + + + private StatsSnapshot getPullRT(final String group, final String topic) { + return this.topicAndGroupPullRT.getStatsDataInMinute(topic + "@" + group); + } + + + private StatsSnapshot getPullTPS(final String group, final String topic) { + return this.topicAndGroupPullTPS.getStatsDataInMinute(topic + "@" + group); + } + + + private StatsSnapshot getConsumeRT(final String group, final String topic) { + StatsSnapshot statsData = this.topicAndGroupConsumeRT.getStatsDataInMinute(topic + "@" + group); + if (0 == statsData.getSum()) { + statsData = this.topicAndGroupConsumeRT.getStatsDataInHour(topic + "@" + group); + } + + return statsData; + } + + + private StatsSnapshot getConsumeOKTPS(final String group, final String topic) { + return this.topicAndGroupConsumeOKTPS.getStatsDataInMinute(topic + "@" + group); + } + + + private StatsSnapshot getConsumeFailedTPS(final String group, final String topic) { + return this.topicAndGroupConsumeFailedTPS.getStatsDataInMinute(topic + "@" + group); + } + + + public ConsumeStatus consumeStatus(final String group, final String topic) { + ConsumeStatus cs = new ConsumeStatus(); + { + StatsSnapshot ss = this.getPullRT(group, topic); + if (ss != null) { + cs.setPullRT(ss.getAvgpt()); + } + } + + { + StatsSnapshot ss = this.getPullTPS(group, topic); + if (ss != null) { + cs.setPullTPS(ss.getTps()); + } + } + + { + StatsSnapshot ss = this.getConsumeRT(group, topic); + if (ss != null) { + cs.setConsumeRT(ss.getAvgpt()); + } + } + + { + StatsSnapshot ss = this.getConsumeOKTPS(group, topic); + if (ss != null) { + cs.setConsumeOKTPS(ss.getTps()); + } + } + + { + StatsSnapshot ss = this.getConsumeFailedTPS(group, topic); + if (ss != null) { + cs.setConsumeFailedTPS(ss.getTps()); + } + } + + { + StatsSnapshot ss = this.topicAndGroupConsumeFailedTPS.getStatsDataInHour(topic + "@" + group); + if (ss != null) { + cs.setConsumeFailedMsgs(ss.getSum()); + } + } + + return cs; + } +} diff --git a/rocketmq-client/src/main/resources/log4j_rocketmq_client.xml b/rocketmq-client/src/main/resources/log4j_rocketmq_client.xml new file mode 100644 index 000000000..5911352a3 --- /dev/null +++ b/rocketmq-client/src/main/resources/log4j_rocketmq_client.xml @@ -0,0 +1,41 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/rocketmq-client/src/main/resources/logback_rocketmq_client.xml b/rocketmq-client/src/main/resources/logback_rocketmq_client.xml new file mode 100644 index 000000000..4f8d78688 --- /dev/null +++ b/rocketmq-client/src/main/resources/logback_rocketmq_client.xml @@ -0,0 +1,41 @@ + + + + ${user.home}/logs/rocketmqlogs/rocketmq_client.log + true + + ${user.home}/logs/rocketmqlogs/otherdays/rocketmq_client.%i.log + + 1 + 10 + + + 100MB + + + %d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n + UTF-8 + + + + + + + + + + + + + + + + + + + + + + diff --git a/rocketmq-client/src/test/java/com/alibaba/rocketmq/client/TestMultiConsumerProducer.java b/rocketmq-client/src/test/java/com/alibaba/rocketmq/client/TestMultiConsumerProducer.java new file mode 100644 index 000000000..4bb2cd9c9 --- /dev/null +++ b/rocketmq-client/src/test/java/com/alibaba/rocketmq/client/TestMultiConsumerProducer.java @@ -0,0 +1,72 @@ +package com.alibaba.rocketmq.client; + +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +import org.junit.Test; + +import com.alibaba.rocketmq.client.consumer.DefaultMQPushConsumer; +import com.alibaba.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext; +import com.alibaba.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus; +import com.alibaba.rocketmq.client.consumer.listener.MessageListenerConcurrently; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.client.producer.DefaultMQProducer; +import com.alibaba.rocketmq.client.producer.SendResult; +import com.alibaba.rocketmq.common.consumer.ConsumeFromWhere; +import com.alibaba.rocketmq.common.message.Message; +import com.alibaba.rocketmq.common.message.MessageExt; + + +public class TestMultiConsumerProducer { + private static final String TOPIC_TEST = "TopicTest-fundmng"; + + + @Test + public void testProducerConsumer() throws MQClientException, InterruptedException { + System.setProperty("rocketmq.namesrv.domain", "jmenv.tbsite.alipay.net"); + + DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("S_fundmng_demo_producer"); + DefaultMQProducer producer = new DefaultMQProducer("P_fundmng_demo_producer"); + + consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET); + consumer.subscribe(TOPIC_TEST, null); + + final AtomicLong lastReceivedMills = new AtomicLong(System.currentTimeMillis()); + + final AtomicLong consumeTimes = new AtomicLong(0); + + consumer.registerMessageListener(new MessageListenerConcurrently() { + public ConsumeConcurrentlyStatus consumeMessage(final List msgs, + final ConsumeConcurrentlyContext context) { + System.out.println("接收了" + consumeTimes.incrementAndGet() + "条消息!"); + + lastReceivedMills.set(System.currentTimeMillis()); + + return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; + } + }); + + consumer.start(); + producer.start(); + + for (int i = 0; i < 100; i++) { + try { + Message msg = new Message(TOPIC_TEST, ("Hello RocketMQ " + i).getBytes()); + SendResult sendResult = producer.send(msg); + System.out.println(sendResult); + } + catch (Exception e) { + TimeUnit.SECONDS.sleep(1); + } + } + + // wait no messages + while ((System.currentTimeMillis() - lastReceivedMills.get()) < 5000) { + TimeUnit.MILLISECONDS.sleep(200); + } + + consumer.shutdown(); + producer.shutdown(); + } +} diff --git a/rocketmq-client/src/test/java/com/alibaba/rocketmq/client/ValidatorsTest.java b/rocketmq-client/src/test/java/com/alibaba/rocketmq/client/ValidatorsTest.java new file mode 100644 index 000000000..93b3cec42 --- /dev/null +++ b/rocketmq-client/src/test/java/com/alibaba/rocketmq/client/ValidatorsTest.java @@ -0,0 +1,23 @@ +package com.alibaba.rocketmq.client; + +import org.junit.Assert; +import org.junit.Test; + + +public class ValidatorsTest { + + @Test + public void testTopic() { + try { + Validators.checkTopic("Hello"); + Validators.checkTopic("%RETRY%Hello"); + Validators.checkTopic("_%RETRY%Hello"); + Validators.checkTopic("-%RETRY%Hello"); + Validators.checkTopic("223-%RETRY%Hello"); + } + catch (Exception e) { + e.printStackTrace(); + Assert.assertTrue(false); + } + } +} diff --git a/rocketmq-client/src/test/java/com/alibaba/rocketmq/client/consumer/loadbalance/AllocateMessageQueueAveragelyTest.java b/rocketmq-client/src/test/java/com/alibaba/rocketmq/client/consumer/loadbalance/AllocateMessageQueueAveragelyTest.java new file mode 100644 index 000000000..e5eb6e230 --- /dev/null +++ b/rocketmq-client/src/test/java/com/alibaba/rocketmq/client/consumer/loadbalance/AllocateMessageQueueAveragelyTest.java @@ -0,0 +1,277 @@ +/* + * @author yubao.fyb@taoboa.com + * @version $id$ + */ +package com.alibaba.rocketmq.client.consumer.loadbalance; + +import java.util.ArrayList; +import java.util.List; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import com.alibaba.rocketmq.client.consumer.AllocateMessageQueueStrategy; +import com.alibaba.rocketmq.client.consumer.rebalance.AllocateMessageQueueAveragely; +import com.alibaba.rocketmq.client.consumer.rebalance.AllocateMessageQueueAveragelyByCircle; +import com.alibaba.rocketmq.common.message.MessageQueue; + + +/** + * @author yubao.fyb@alibaba-inc.com created on 2013-07-03 16:24 + */ +public class AllocateMessageQueueAveragelyTest { + private AllocateMessageQueueStrategy allocateMessageQueueAveragely; + private String currentCID; + private String topic; + private List messageQueueList; + private List consumerIdList; + + + public void createMessageQueueList(int size) { + messageQueueList = new ArrayList(size); + for (int i = 0; i < size; i++) { + MessageQueue mq = new MessageQueue(topic, "brokerName", i); + messageQueueList.add(mq); + } + } + + + public void createConsumerIdList(int size) { + consumerIdList = new ArrayList(size); + for (int i = 0; i < size; i++) { + consumerIdList.add(String.valueOf(i)); + } + } + + + @Before + public void init() { + allocateMessageQueueAveragely = new AllocateMessageQueueAveragely(); + topic = "topic_test"; + } + + + @Test + public void testConsumer1() { // consumerList大小为1 + currentCID = "0"; + createConsumerIdList(1); + createMessageQueueList(5); + List result = + allocateMessageQueueAveragely.allocate("", currentCID, messageQueueList, consumerIdList); + printMessageQueue(result, "testConsumer1"); + Assert.assertEquals(result.size(), 5); + Assert.assertEquals(result.containsAll(getMessageQueueList()), true); + } + + + @Test + public void testConsumer2() { // consumerList大小为2 + currentCID = "1"; + createConsumerIdList(2); + createMessageQueueList(5); + List result = + allocateMessageQueueAveragely.allocate("", currentCID, messageQueueList, consumerIdList); + printMessageQueue(result, "testConsumer2"); + Assert.assertEquals(result.size(), 3); + Assert.assertEquals(result.containsAll(getMessageQueueList().subList(2, 5)), true); + + } + + + @Test + public void testConsumer3CurrentCID0() { // consumerList大小为3 + currentCID = "0"; + createConsumerIdList(3); + createMessageQueueList(5); + List result = + allocateMessageQueueAveragely.allocate("", currentCID, messageQueueList, consumerIdList); + printMessageQueue(result, "testConsumer3CurrentCID0"); + Assert.assertEquals(result.size(), 1); + Assert.assertEquals(result.containsAll(getMessageQueueList().subList(0, 1)), true); + } + + + @Test + public void testConsumer3CurrentCID1() { // consumerList大小为3 + currentCID = "1"; + createConsumerIdList(3); + createMessageQueueList(5); + List result = + allocateMessageQueueAveragely.allocate("", currentCID, messageQueueList, consumerIdList); + printMessageQueue(result, "testConsumer3CurrentCID1"); + Assert.assertEquals(result.size(), 1); + Assert.assertEquals(result.containsAll(getMessageQueueList().subList(1, 2)), true); + } + + + @Test + public void testConsumer3CurrentCID2() { // consumerList大小为3 + currentCID = "2"; + createConsumerIdList(3); + createMessageQueueList(5); + List result = + allocateMessageQueueAveragely.allocate("", currentCID, messageQueueList, consumerIdList); + printMessageQueue(result, "testConsumer3CurrentCID2"); + Assert.assertEquals(result.size(), 3); + Assert.assertEquals(result.containsAll(getMessageQueueList().subList(2, 5)), true); + } + + + @Test + public void testConsumer4() { // consumerList大小为4 + currentCID = "1"; + createConsumerIdList(4); + createMessageQueueList(5); + List result = + allocateMessageQueueAveragely.allocate("", currentCID, messageQueueList, consumerIdList); + printMessageQueue(result, "testConsumer4"); + Assert.assertEquals(result.size(), 1); + Assert.assertEquals(result.containsAll(getMessageQueueList().subList(1, 2)), true); + } + + + @Test + public void testConsumer5() { // consumerList大小为5 + currentCID = "1"; + createConsumerIdList(5); + createMessageQueueList(5); + List result = + allocateMessageQueueAveragely.allocate("", currentCID, messageQueueList, consumerIdList); + printMessageQueue(result, "testConsumer5"); + Assert.assertEquals(result.size(), 1); + Assert.assertEquals(result.containsAll(getMessageQueueList().subList(1, 2)), true); + } + + + @Test + public void testConsumer6() { // consumerList大小为6 + currentCID = "1"; + createConsumerIdList(2); + createMessageQueueList(6); + List result = + allocateMessageQueueAveragely.allocate("", currentCID, messageQueueList, consumerIdList); + printMessageQueue(result, "testConsumer"); + Assert.assertEquals(result.size(), 3); + Assert.assertEquals(result.containsAll(getMessageQueueList().subList(3, 6)), true); + } + + + @Test + public void testCurrentCIDNotExists() { // CurrentCID不存在 + currentCID = String.valueOf(Integer.MAX_VALUE); + createConsumerIdList(2); + createMessageQueueList(6); + List result = + allocateMessageQueueAveragely.allocate("", currentCID, messageQueueList, consumerIdList); + printMessageQueue(result, "testCurrentCIDNotExists"); + Assert.assertEquals(result.size(), 0); + } + + + @Test(expected = IllegalArgumentException.class) + public void testCurrentCIDIllegalArgument() { // currentCID是空 + createConsumerIdList(2); + createMessageQueueList(6); + allocateMessageQueueAveragely.allocate("", "", getMessageQueueList(), getConsumerIdList()); + } + + + @Test(expected = IllegalArgumentException.class) + public void testMessageQueueIllegalArgument() { // MessageQueue为空 + currentCID = "0"; + createConsumerIdList(2); + allocateMessageQueueAveragely.allocate("", currentCID, null, getConsumerIdList()); + } + + + @Test(expected = IllegalArgumentException.class) + public void testConsumerIdIllegalArgument() { // ConsumerIdList为空 + currentCID = "0"; + createMessageQueueList(6); + allocateMessageQueueAveragely.allocate("", currentCID, getMessageQueueList(), null); + } + + + public List getMessageQueueList() { + return messageQueueList; + } + + + public void setMessageQueueList(List messageQueueList) { + this.messageQueueList = messageQueueList; + } + + + public List getConsumerIdList() { + return consumerIdList; + } + + + public void setConsumerIdList(List consumerIdList) { + this.consumerIdList = consumerIdList; + } + + + public void printMessageQueue(List messageQueueList, String name) { + if (messageQueueList == null || messageQueueList.size() < 1) + return; + System.out.println(name + ".......................................start"); + for (MessageQueue messageQueue : messageQueueList) { + System.out.println(messageQueue); + } + System.out.println(name + ".......................................end"); + } + + + @Test + public void testAllocate() { + AllocateMessageQueueAveragely allocateMessageQueueAveragely = new AllocateMessageQueueAveragely(); + String topic = "topic_test"; + String currentCID = "CID"; + int queueSize = 19; + int consumerSize = 10; + List mqAll = new ArrayList(); + for (int i = 0; i < queueSize; i++) { + MessageQueue mq = new MessageQueue(topic, "brokerName", i); + mqAll.add(mq); + } + + List cidAll = new ArrayList(); + for (int j = 0; j < consumerSize; j++) { + cidAll.add("CID" + j); + } + System.out.println(mqAll.toString()); + System.out.println(cidAll.toString()); + for (int i = 0; i < consumerSize; i++) { + List rs = allocateMessageQueueAveragely.allocate("", currentCID + i, mqAll, cidAll); + System.out.println("rs[" + currentCID + i + "]:" + rs.toString()); + } + } + + + @Test + public void testAllocateByCircle() { + AllocateMessageQueueAveragelyByCircle circle = new AllocateMessageQueueAveragelyByCircle(); + String topic = "topic_test"; + String currentCID = "CID"; + int consumerSize = 3; + int queueSize = 13; + List mqAll = new ArrayList(); + for (int i = 0; i < queueSize; i++) { + MessageQueue mq = new MessageQueue(topic, "brokerName", i); + mqAll.add(mq); + } + + List cidAll = new ArrayList(); + for (int j = 0; j < consumerSize; j++) { + cidAll.add("CID" + j); + } + System.out.println(mqAll.toString()); + System.out.println(cidAll.toString()); + for (int i = 0; i < consumerSize; i++) { + List rs = circle.allocate("", currentCID + i, mqAll, cidAll); + System.out.println("rs[" + currentCID + i + "]:" + rs.toString()); + } + } +} diff --git a/rocketmq-common/pom.xml b/rocketmq-common/pom.xml new file mode 100644 index 000000000..df7c6aa1b --- /dev/null +++ b/rocketmq-common/pom.xml @@ -0,0 +1,26 @@ + + + com.alibaba.rocketmq + rocketmq-all + 3.2.2 + + + 4.0.0 + jar + rocketmq-common + rocketmq-common ${project.version} + + + + + junit + junit + test + + + ${project.groupId} + rocketmq-remoting + + + diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/BrokerConfig.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/BrokerConfig.java new file mode 100644 index 000000000..a0ed68fa2 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/BrokerConfig.java @@ -0,0 +1,375 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common; + +import com.alibaba.rocketmq.common.annotation.ImportantField; +import com.alibaba.rocketmq.common.constant.PermName; +import com.alibaba.rocketmq.remoting.common.RemotingUtil; + +import java.net.InetAddress; +import java.net.UnknownHostException; + + +/** + * 服务器配置 + * + * @author shijia.wxr + */ +public class BrokerConfig { + private String rocketmqHome = System.getProperty(MixAll.ROCKETMQ_HOME_PROPERTY, + System.getenv(MixAll.ROCKETMQ_HOME_ENV)); + @ImportantField + private String namesrvAddr = System.getProperty(MixAll.NAMESRV_ADDR_PROPERTY, + System.getenv(MixAll.NAMESRV_ADDR_ENV)); + @ImportantField + private String brokerIP1 = RemotingUtil.getLocalAddress(); + private String brokerIP2 = RemotingUtil.getLocalAddress(); + @ImportantField + private String brokerName = localHostName(); + @ImportantField + private String brokerClusterName = "DefaultCluster"; + @ImportantField + private long brokerId = MixAll.MASTER_ID; + private int brokerPermission = PermName.PERM_READ | PermName.PERM_WRITE; + private int defaultTopicQueueNums = 8; + // 自动创建Topic功能是否开启（线上建议关闭） + @ImportantField + private boolean autoCreateTopicEnable = true; + // 自动创建以集群名字命名的Topic功能是否开启 + private boolean clusterTopicEnable = true; + // 自动创建以服务器名字命名的Topic功能是否开启 + private boolean brokerTopicEnable = true; + // 自动创建订阅组功能是否开启（线上建议关闭） + @ImportantField + private boolean autoCreateSubscriptionGroup = true; + + private int sendMessageThreadPoolNums = 16 + Runtime.getRuntime().availableProcessors() * 4; + private int pullMessageThreadPoolNums = 16 + Runtime.getRuntime().availableProcessors() * 2; + private int adminBrokerThreadPoolNums = 16; + private int clientManageThreadPoolNums = 16; + + private int flushConsumerOffsetInterval = 1000 * 5; + + private int flushConsumerOffsetHistoryInterval = 1000 * 60; + + // 是否拒绝接收事务消息 + @ImportantField + private boolean rejectTransactionMessage = false; + + // 是否从地址服务器寻找Name Server地址，正式发布后，默认值为false + @ImportantField + private boolean fetchNamesrvAddrByAddressServer = false; + + // 发送消息对应的线程池阻塞队列size + private int sendThreadPoolQueueCapacity = 100000; + + // 订阅消息对应的线程池阻塞队列size + private int pullThreadPoolQueueCapacity = 100000; + + // 过滤服务器数量 + private int filterServerNums = 0; + + // Consumer订阅消息时，Broker是否开启长轮询 + private boolean longPollingEnable = true; + + // 如果是短轮询，服务器挂起时间 + private long shortPollingTimeMills = 1000; + + // notify consumerId changed 开关 + private boolean notifyConsumerIdsChangedEnable = true; + + + public static String localHostName() { + try { + return InetAddress.getLocalHost().getHostName(); + } + catch (UnknownHostException e) { + e.printStackTrace(); + } + + return "DEFAULT_BROKER"; + } + + + public String getRocketmqHome() { + return rocketmqHome; + } + + + public void setRocketmqHome(String rocketmqHome) { + this.rocketmqHome = rocketmqHome; + } + + + public String getBrokerName() { + return brokerName; + } + + + public void setBrokerName(String brokerName) { + this.brokerName = brokerName; + } + + + public int getBrokerPermission() { + return brokerPermission; + } + + + public void setBrokerPermission(int brokerPermission) { + this.brokerPermission = brokerPermission; + } + + + public int getDefaultTopicQueueNums() { + return defaultTopicQueueNums; + } + + + public void setDefaultTopicQueueNums(int defaultTopicQueueNums) { + this.defaultTopicQueueNums = defaultTopicQueueNums; + } + + + public boolean isAutoCreateTopicEnable() { + return autoCreateTopicEnable; + } + + + public void setAutoCreateTopicEnable(boolean autoCreateTopic) { + this.autoCreateTopicEnable = autoCreateTopic; + } + + + public String getBrokerClusterName() { + return brokerClusterName; + } + + + public void setBrokerClusterName(String brokerClusterName) { + this.brokerClusterName = brokerClusterName; + } + + + public String getBrokerIP1() { + return brokerIP1; + } + + + public void setBrokerIP1(String brokerIP1) { + this.brokerIP1 = brokerIP1; + } + + + public String getBrokerIP2() { + return brokerIP2; + } + + + public void setBrokerIP2(String brokerIP2) { + this.brokerIP2 = brokerIP2; + } + + + public int getSendMessageThreadPoolNums() { + return sendMessageThreadPoolNums; + } + + + public void setSendMessageThreadPoolNums(int sendMessageThreadPoolNums) { + this.sendMessageThreadPoolNums = sendMessageThreadPoolNums; + } + + + public int getPullMessageThreadPoolNums() { + return pullMessageThreadPoolNums; + } + + + public void setPullMessageThreadPoolNums(int pullMessageThreadPoolNums) { + this.pullMessageThreadPoolNums = pullMessageThreadPoolNums; + } + + + public int getAdminBrokerThreadPoolNums() { + return adminBrokerThreadPoolNums; + } + + + public void setAdminBrokerThreadPoolNums(int adminBrokerThreadPoolNums) { + this.adminBrokerThreadPoolNums = adminBrokerThreadPoolNums; + } + + + public int getFlushConsumerOffsetInterval() { + return flushConsumerOffsetInterval; + } + + + public void setFlushConsumerOffsetInterval(int flushConsumerOffsetInterval) { + this.flushConsumerOffsetInterval = flushConsumerOffsetInterval; + } + + + public int getFlushConsumerOffsetHistoryInterval() { + return flushConsumerOffsetHistoryInterval; + } + + + public void setFlushConsumerOffsetHistoryInterval(int flushConsumerOffsetHistoryInterval) { + this.flushConsumerOffsetHistoryInterval = flushConsumerOffsetHistoryInterval; + } + + + public boolean isClusterTopicEnable() { + return clusterTopicEnable; + } + + + public void setClusterTopicEnable(boolean clusterTopicEnable) { + this.clusterTopicEnable = clusterTopicEnable; + } + + + public String getNamesrvAddr() { + return namesrvAddr; + } + + + public void setNamesrvAddr(String namesrvAddr) { + this.namesrvAddr = namesrvAddr; + } + + + public long getBrokerId() { + return brokerId; + } + + + public void setBrokerId(long brokerId) { + this.brokerId = brokerId; + } + + + public boolean isAutoCreateSubscriptionGroup() { + return autoCreateSubscriptionGroup; + } + + + public void setAutoCreateSubscriptionGroup(boolean autoCreateSubscriptionGroup) { + this.autoCreateSubscriptionGroup = autoCreateSubscriptionGroup; + } + + + public boolean isRejectTransactionMessage() { + return rejectTransactionMessage; + } + + + public void setRejectTransactionMessage(boolean rejectTransactionMessage) { + this.rejectTransactionMessage = rejectTransactionMessage; + } + + + public boolean isFetchNamesrvAddrByAddressServer() { + return fetchNamesrvAddrByAddressServer; + } + + + public void setFetchNamesrvAddrByAddressServer(boolean fetchNamesrvAddrByAddressServer) { + this.fetchNamesrvAddrByAddressServer = fetchNamesrvAddrByAddressServer; + } + + + public int getSendThreadPoolQueueCapacity() { + return sendThreadPoolQueueCapacity; + } + + + public void setSendThreadPoolQueueCapacity(int sendThreadPoolQueueCapacity) { + this.sendThreadPoolQueueCapacity = sendThreadPoolQueueCapacity; + } + + + public int getPullThreadPoolQueueCapacity() { + return pullThreadPoolQueueCapacity; + } + + + public void setPullThreadPoolQueueCapacity(int pullThreadPoolQueueCapacity) { + this.pullThreadPoolQueueCapacity = pullThreadPoolQueueCapacity; + } + + + public boolean isBrokerTopicEnable() { + return brokerTopicEnable; + } + + + public void setBrokerTopicEnable(boolean brokerTopicEnable) { + this.brokerTopicEnable = brokerTopicEnable; + } + + + public int getFilterServerNums() { + return filterServerNums; + } + + + public void setFilterServerNums(int filterServerNums) { + this.filterServerNums = filterServerNums; + } + + + public boolean isLongPollingEnable() { + return longPollingEnable; + } + + + public void setLongPollingEnable(boolean longPollingEnable) { + this.longPollingEnable = longPollingEnable; + } + + + public boolean isNotifyConsumerIdsChangedEnable() { + return notifyConsumerIdsChangedEnable; + } + + + public void setNotifyConsumerIdsChangedEnable(boolean notifyConsumerIdsChangedEnable) { + this.notifyConsumerIdsChangedEnable = notifyConsumerIdsChangedEnable; + } + + + public long getShortPollingTimeMills() { + return shortPollingTimeMills; + } + + + public void setShortPollingTimeMills(long shortPollingTimeMills) { + this.shortPollingTimeMills = shortPollingTimeMills; + } + + + public int getClientManageThreadPoolNums() { + return clientManageThreadPoolNums; + } + + + public void setClientManageThreadPoolNums(int clientManageThreadPoolNums) { + this.clientManageThreadPoolNums = clientManageThreadPoolNums; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/ConfigManager.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/ConfigManager.java new file mode 100644 index 000000000..3e88339bd --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/ConfigManager.java @@ -0,0 +1,102 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common; + +import java.io.IOException; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.rocketmq.common.constant.LoggerName; + + +/** + * 各种配置的管理接口 + * + * @author shijia.wxr + * @since 2013-6-18 + */ +public abstract class ConfigManager { + private static final Logger plog = LoggerFactory.getLogger(LoggerName.CommonLoggerName); + + + public abstract String encode(); + + + public abstract String encode(final boolean prettyFormat); + + + public abstract void decode(final String jsonString); + + + public abstract String configFilePath(); + + + public boolean load() { + String fileName = null; + try { + fileName = this.configFilePath(); + String jsonString = MixAll.file2String(fileName); + // 文件不存在，或者为空文件 + if (null == jsonString || jsonString.length() == 0) { + return this.loadBak(); + } + else { + this.decode(jsonString); + plog.info("load {} OK", fileName); + return true; + } + } + catch (Exception e) { + plog.error("load " + fileName + " Failed, and try to load backup file", e); + return this.loadBak(); + } + } + + + private boolean loadBak() { + String fileName = null; + try { + fileName = this.configFilePath(); + String jsonString = MixAll.file2String(fileName + ".bak"); + if (jsonString != null && jsonString.length() > 0) { + this.decode(jsonString); + plog.info("load " + fileName + " OK"); + return true; + } + } + catch (Exception e) { + plog.error("load " + fileName + " Failed", e); + return false; + } + + return true; + } + + + public synchronized void persist() { + String jsonString = this.encode(true); + if (jsonString != null) { + String fileName = this.configFilePath(); + try { + MixAll.string2File(jsonString, fileName); + } + catch (IOException e) { + plog.error("persist file Exception, " + fileName, e); + } + } + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/DataVersion.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/DataVersion.java new file mode 100644 index 000000000..b5bc747e4 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/DataVersion.java @@ -0,0 +1,70 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common; + +import java.util.concurrent.atomic.AtomicLong; + +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; + + +/** + * 用来标识数据的版本号 + * + * @author shijia.wxr + */ +public class DataVersion extends RemotingSerializable { + private long timestatmp = System.currentTimeMillis(); + private AtomicLong counter = new AtomicLong(0); + + + public void assignNewOne(final DataVersion dataVersion) { + this.timestatmp = dataVersion.timestatmp; + this.counter.set(dataVersion.counter.get()); + } + + + public void nextVersion() { + this.timestatmp = System.currentTimeMillis(); + this.counter.incrementAndGet(); + } + + + public long getTimestatmp() { + return timestatmp; + } + + + public void setTimestatmp(long timestatmp) { + this.timestatmp = timestatmp; + } + + + public AtomicLong getCounter() { + return counter; + } + + + public void setCounter(AtomicLong counter) { + this.counter = counter; + } + + + @Override + public boolean equals(Object obj) { + DataVersion dv = (DataVersion) obj; + return this.timestatmp == dv.timestatmp && this.counter.get() == dv.counter.get(); + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/MQVersion.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/MQVersion.java new file mode 100644 index 000000000..ab20ce1b3 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/MQVersion.java @@ -0,0 +1,238 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common; + +/** + * 定义各个版本信息 + * + * @author shijia.wxr + */ +public class MQVersion { + // TODO 每次发布版本都要修改此处版本号 + public static final int CurrentVersion = Version.V3_2_2_SNAPSHOT.ordinal(); + + + public static String getVersionDesc(int value) { + try { + Version v = Version.values()[value]; + return v.name(); + } + catch (Exception e) { + } + + return "HigherVersion"; + } + + + public static Version value2Version(int value) { + return Version.values()[value]; + } + + public static enum Version { + V3_0_0_SNAPSHOT, + V3_0_0_ALPHA1, + V3_0_0_BETA1, + V3_0_0_BETA2, + V3_0_0_BETA3, + V3_0_0_BETA4, + V3_0_0_BETA5, + V3_0_0_BETA6_SNAPSHOT, + V3_0_0_BETA6, + V3_0_0_BETA7_SNAPSHOT, + V3_0_0_BETA7, + V3_0_0_BETA8_SNAPSHOT, + V3_0_0_BETA8, + V3_0_0_BETA9_SNAPSHOT, + V3_0_0_BETA9, + V3_0_0_FINAL, + V3_0_1_SNAPSHOT, + V3_0_1, + V3_0_2_SNAPSHOT, + V3_0_2, + V3_0_3_SNAPSHOT, + V3_0_3, + V3_0_4_SNAPSHOT, + V3_0_4, + V3_0_5_SNAPSHOT, + V3_0_5, + V3_0_6_SNAPSHOT, + V3_0_6, + V3_0_7_SNAPSHOT, + V3_0_7, + V3_0_8_SNAPSHOT, + V3_0_8, + V3_0_9_SNAPSHOT, + V3_0_9, + + V3_0_10_SNAPSHOT, + V3_0_10, + + V3_0_11_SNAPSHOT, + V3_0_11, + + V3_0_12_SNAPSHOT, + V3_0_12, + + V3_0_13_SNAPSHOT, + V3_0_13, + + V3_0_14_SNAPSHOT, + V3_0_14, + + V3_0_15_SNAPSHOT, + V3_0_15, + + V3_1_0_SNAPSHOT, + V3_1_0, + + V3_1_1_SNAPSHOT, + V3_1_1, + + V3_1_2_SNAPSHOT, + V3_1_2, + + V3_1_3_SNAPSHOT, + V3_1_3, + + V3_1_4_SNAPSHOT, + V3_1_4, + + V3_1_5_SNAPSHOT, + V3_1_5, + + V3_1_6_SNAPSHOT, + V3_1_6, + + V3_1_7_SNAPSHOT, + V3_1_7, + + V3_1_8_SNAPSHOT, + V3_1_8, + + V3_1_9_SNAPSHOT, + V3_1_9, + + V3_2_0_SNAPSHOT, + V3_2_0, + + V3_2_1_SNAPSHOT, + V3_2_1, + + V3_2_2_SNAPSHOT, + V3_2_2, + + V3_2_3_SNAPSHOT, + V3_2_3, + + V3_2_4_SNAPSHOT, + V3_2_4, + + V3_2_5_SNAPSHOT, + V3_2_5, + + V3_2_6_SNAPSHOT, + V3_2_6, + + V3_2_7_SNAPSHOT, + V3_2_7, + + V3_2_8_SNAPSHOT, + V3_2_8, + + V3_2_9_SNAPSHOT, + V3_2_9, + + V3_3_1_SNAPSHOT, + V3_3_1, + + V3_3_2_SNAPSHOT, + V3_3_2, + + V3_3_3_SNAPSHOT, + V3_3_3, + + V3_3_4_SNAPSHOT, + V3_3_4, + + V3_3_5_SNAPSHOT, + V3_3_5, + + V3_3_6_SNAPSHOT, + V3_3_6, + + V3_3_7_SNAPSHOT, + V3_3_7, + + V3_3_8_SNAPSHOT, + V3_3_8, + + V3_3_9_SNAPSHOT, + V3_3_9, + + V3_4_1_SNAPSHOT, + V3_4_1, + + V3_4_2_SNAPSHOT, + V3_4_2, + + V3_4_3_SNAPSHOT, + V3_4_3, + + V3_4_4_SNAPSHOT, + V3_4_4, + + V3_4_5_SNAPSHOT, + V3_4_5, + + V3_4_6_SNAPSHOT, + V3_4_6, + + V3_4_7_SNAPSHOT, + V3_4_7, + + V3_4_8_SNAPSHOT, + V3_4_8, + + V3_4_9_SNAPSHOT, + V3_4_9, + V3_5_1_SNAPSHOT, + V3_5_1, + + V3_5_2_SNAPSHOT, + V3_5_2, + + V3_5_3_SNAPSHOT, + V3_5_3, + + V3_5_4_SNAPSHOT, + V3_5_4, + + V3_5_5_SNAPSHOT, + V3_5_5, + + V3_5_6_SNAPSHOT, + V3_5_6, + + V3_5_7_SNAPSHOT, + V3_5_7, + + V3_5_8_SNAPSHOT, + V3_5_8, + + V3_5_9_SNAPSHOT, + V3_5_9, + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/MixAll.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/MixAll.java new file mode 100644 index 000000000..1ee933b53 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/MixAll.java @@ -0,0 +1,484 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common; + +import java.io.*; +import java.lang.annotation.Annotation; +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.net.*; +import java.nio.ByteBuffer; +import java.util.*; +import java.util.concurrent.atomic.AtomicLong; + +import org.slf4j.Logger; + +import com.alibaba.rocketmq.common.annotation.ImportantField; + + +/** + * 各种方法大杂烩 + * + * @author shijia.wxr + * @author lansheng.zj@taobao.com + */ +public class MixAll { + public static final String ROCKETMQ_HOME_ENV = "ROCKETMQ_HOME"; + public static final String ROCKETMQ_HOME_PROPERTY = "rocketmq.home.dir"; + public static final String NAMESRV_ADDR_ENV = "NAMESRV_ADDR"; + public static final String NAMESRV_ADDR_PROPERTY = "rocketmq.namesrv.addr"; + public static final String MESSAGE_COMPRESS_LEVEL = "rocketmq.message.compressLevel"; + public static final String WS_DOMAIN_NAME = System.getProperty("rocketmq.namesrv.domain", + "jmenv.tbsite.net"); + public static final String WS_DOMAIN_SUBGROUP = System.getProperty("rocketmq.namesrv.domain.subgroup", + "nsaddr"); + // http://jmenv.tbsite.net:8080/rocketmq/nsaddr + public static final String WS_ADDR = "http://" + WS_DOMAIN_NAME + ":8080/rocketmq/" + WS_DOMAIN_SUBGROUP; + public static final String DEFAULT_TOPIC = "TBW102"; + public static final String BENCHMARK_TOPIC = "BenchmarkTest"; + public static final String DEFAULT_PRODUCER_GROUP = "DEFAULT_PRODUCER"; + public static final String DEFAULT_CONSUMER_GROUP = "DEFAULT_CONSUMER"; + public static final String TOOLS_CONSUMER_GROUP = "TOOLS_CONSUMER"; + public static final String FILTERSRV_CONSUMER_GROUP = "FILTERSRV_CONSUMER"; + public static final String MONITOR_CONSUMER_GROUP = "__MONITOR_CONSUMER"; + public static final String CLIENT_INNER_PRODUCER_GROUP = "CLIENT_INNER_PRODUCER"; + public static final String SELF_TEST_TOPIC = "SELF_TEST_TOPIC"; + public static final String OFFSET_MOVED_EVENT = "OFFSET_MOVED_EVENT"; + + public static final List LocalInetAddrs = getLocalInetAddress(); + public static final String Localhost = localhost(); + public static final String DEFAULT_CHARSET = "UTF-8"; + public static final long MASTER_ID = 0L; + public static final long CURRENT_JVM_PID = getPID(); + // 为每个Consumer Group建立一个默认的Topic，前缀 + GroupName，用来保存处理失败需要重试的消息 + public static final String RETRY_GROUP_TOPIC_PREFIX = "%RETRY%"; + // 为每个Consumer Group建立一个默认的Topic，前缀 + GroupName，用来保存重试多次都失败，接下来不再重试的消息 + public static final String DLQ_GROUP_TOPIC_PREFIX = "%DLQ%"; + + + public static String getRetryTopic(final String consumerGroup) { + return RETRY_GROUP_TOPIC_PREFIX + consumerGroup; + } + + + public static String getDLQTopic(final String consumerGroup) { + return DLQ_GROUP_TOPIC_PREFIX + consumerGroup; + } + + + public static long getPID() { + String processName = java.lang.management.ManagementFactory.getRuntimeMXBean().getName(); + if (processName != null && processName.length() > 0) { + try { + return Long.parseLong(processName.split("@")[0]); + } + catch (Exception e) { + return 0; + } + } + + return 0; + } + + + public static long createBrokerId(final String ip, final int port) { + InetSocketAddress isa = new InetSocketAddress(ip, port); + byte[] ipArray = isa.getAddress().getAddress(); + ByteBuffer bb = ByteBuffer.allocate(8); + bb.put(ipArray); + bb.putInt(port); + long value = bb.getLong(0); + return Math.abs(value); + } + + + /** + * 安全的写文件 + */ + public static final void string2File(final String str, final String fileName) throws IOException { + // 先写入临时文件 + String tmpFile = fileName + ".tmp"; + string2FileNotSafe(str, tmpFile); + + // 备份之前的文件 + String bakFile = fileName + ".bak"; + String prevContent = file2String(fileName); + if (prevContent != null) { + string2FileNotSafe(prevContent, bakFile); + } + + // 删除正式文件 + File file = new File(fileName); + file.delete(); + + // 临时文件改为正式文件 + file = new File(tmpFile); + file.renameTo(new File(fileName)); + } + + + public static final void string2FileNotSafe(final String str, final String fileName) throws IOException { + File file = new File(fileName); + File fileParent = file.getParentFile(); + if (fileParent != null) { + fileParent.mkdirs(); + } + FileWriter fileWriter = null; + + try { + fileWriter = new FileWriter(file); + fileWriter.write(str); + } + catch (IOException e) { + throw e; + } + finally { + if (fileWriter != null) { + try { + fileWriter.close(); + } + catch (IOException e) { + throw e; + } + } + } + } + + + public static final String file2String(final String fileName) { + File file = new File(fileName); + return file2String(file); + } + + + public static final String file2String(final URL url) { + InputStream in = null; + try { + URLConnection urlConnection = url.openConnection(); + urlConnection.setUseCaches(false); + in = urlConnection.getInputStream(); + int len = in.available(); + byte[] data = new byte[len]; + in.read(data, 0, len); + return new String(data, "UTF-8"); + } + catch (Exception e) { + } + finally { + if (null != in) { + try { + in.close(); + } + catch (IOException e) { + } + } + } + + return null; + } + + + private static final String file2String(final File file) { + if (file.exists()) { + char[] data = new char[(int) file.length()]; + boolean result = false; + + FileReader fileReader = null; + try { + fileReader = new FileReader(file); + int len = fileReader.read(data); + result = (len == data.length); + } + catch (IOException e) { + // e.printStackTrace(); + } + finally { + if (fileReader != null) { + try { + fileReader.close(); + } + catch (IOException e) { + e.printStackTrace(); + } + } + } + + if (result) { + String value = new String(data); + return value; + } + } + return null; + } + + + public static String findClassPath(Class c) { + URL url = c.getProtectionDomain().getCodeSource().getLocation(); + return url.getPath(); + } + + + public static void printObjectProperties(final Logger log, final Object object) { + printObjectProperties(log, object, false); + } + + + public static void printObjectProperties(final Logger log, final Object object, + final boolean onlyImportantField) { + Field[] fields = object.getClass().getDeclaredFields(); + for (Field field : fields) { + if (!Modifier.isStatic(field.getModifiers())) { + String name = field.getName(); + if (!name.startsWith("this")) { + Object value = null; + try { + field.setAccessible(true); + value = field.get(object); + if (null == value) { + value = ""; + } + } + catch (IllegalArgumentException e) { + System.out.println(e); + } + catch (IllegalAccessException e) { + System.out.println(e); + } + + if (onlyImportantField) { + Annotation annotation = field.getAnnotation(ImportantField.class); + if (null == annotation) { + continue; + } + } + + if (log != null) { + log.info(name + "=" + value); + } + else { + System.out.println(name + "=" + value); + } + } + } + } + } + + + public static String properties2String(final Properties properties) { + Set sets = properties.keySet(); + StringBuilder sb = new StringBuilder(); + for (Object key : sets) { + Object value = properties.get(key); + if (value != null) { + sb.append(key.toString() + "=" + value.toString() + "\n"); + } + } + + return sb.toString(); + } + + + /** + * 字符串转化成Properties 字符串和Properties配置文件格式一样 + */ + public static Properties string2Properties(final String str) { + Properties properties = new Properties(); + try { + InputStream in = new ByteArrayInputStream(str.getBytes(DEFAULT_CHARSET)); + properties.load(in); + } + catch (UnsupportedEncodingException e) { + e.printStackTrace(); + return null; + } + catch (IOException e) { + e.printStackTrace(); + return null; + } + + return properties; + } + + + /** + * 将对象各成员属性值转化为Properties + */ + public static Properties object2Properties(final Object object) { + Properties properties = new Properties(); + + Field[] fields = object.getClass().getDeclaredFields(); + for (Field field : fields) { + if (!Modifier.isStatic(field.getModifiers())) { + String name = field.getName(); + if (!name.startsWith("this")) { + Object value = null; + try { + field.setAccessible(true); + value = field.get(object); + } + catch (IllegalArgumentException e) { + System.out.println(e); + } + catch (IllegalAccessException e) { + System.out.println(e); + } + + if (value != null) { + properties.setProperty(name, value.toString()); + } + } + } + } + + return properties; + } + + + /** + * 将Properties中的值写入Object + */ + public static void properties2Object(final Properties p, final Object object) { + Method[] methods = object.getClass().getMethods(); + for (Method method : methods) { + String mn = method.getName(); + if (mn.startsWith("set")) { + try { + String tmp = mn.substring(4); + String first = mn.substring(3, 4); + + String key = first.toLowerCase() + tmp; + String property = p.getProperty(key); + if (property != null) { + Class[] pt = method.getParameterTypes(); + if (pt != null && pt.length > 0) { + String cn = pt[0].getSimpleName(); + Object arg = null; + if (cn.equals("int")) { + arg = Integer.parseInt(property); + } + else if (cn.equals("long")) { + arg = Long.parseLong(property); + } + else if (cn.equals("double")) { + arg = Double.parseDouble(property); + } + else if (cn.equals("boolean")) { + arg = Boolean.parseBoolean(property); + } + else if (cn.equals("String")) { + arg = property; + } + else { + continue; + } + method.invoke(object, new Object[] { arg }); + } + } + } + catch (Throwable e) { + } + } + } + } + + + public static boolean isPropertiesEqual(final Properties p1, final Properties p2) { + return p1.equals(p2); + } + + + public static List getLocalInetAddress() { + List inetAddressList = new ArrayList(); + try { + Enumeration enumeration = NetworkInterface.getNetworkInterfaces(); + while (enumeration.hasMoreElements()) { + NetworkInterface networkInterface = enumeration.nextElement(); + Enumeration addrs = networkInterface.getInetAddresses(); + while (addrs.hasMoreElements()) { + inetAddressList.add(addrs.nextElement().getHostAddress()); + } + } + } + catch (SocketException e) { + throw new RuntimeException("get local inet address fail", e); + } + + return inetAddressList; + } + + + public static boolean isLocalAddr(String address) { + for (String addr : LocalInetAddrs) { + if (address.contains(addr)) + return true; + } + return false; + } + + + private static String localhost() { + try { + InetAddress addr = InetAddress.getLocalHost(); + return addr.getHostAddress(); + } + catch (UnknownHostException e) { + throw new RuntimeException("get localhost fail", e); + } + } + + + public static boolean compareAndIncreaseOnly(final AtomicLong target, final long value) { + long prev = target.get(); + while (value > prev) { + boolean updated = target.compareAndSet(prev, value); + if (updated) + return true; + + prev = target.get(); + } + + return false; + } + + + public Set list2Set(List values) { + Set result = new HashSet(); + for (String v : values) { + result.add(v); + } + return result; + } + + + public List set2List(Set values) { + List result = new ArrayList(); + for (String v : values) { + result.add(v); + } + return result; + } + + + public static String localhostName() { + try { + return InetAddress.getLocalHost().getHostName(); + } + catch (UnknownHostException e) { + throw new RuntimeException("get localhost fail", e); + } + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/Pair.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/Pair.java new file mode 100644 index 000000000..f93d4bebd --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/Pair.java @@ -0,0 +1,50 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common; + +/** + * @author shijia.wxr + */ +public class Pair { + private T1 object1; + private T2 object2; + + + public Pair(T1 object1, T2 object2) { + this.object1 = object1; + this.object2 = object2; + } + + + public T1 getObject1() { + return object1; + } + + + public void setObject1(T1 object1) { + this.object1 = object1; + } + + + public T2 getObject2() { + return object2; + } + + + public void setObject2(T2 object2) { + this.object2 = object2; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/ServiceState.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/ServiceState.java new file mode 100644 index 000000000..4fc022416 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/ServiceState.java @@ -0,0 +1,40 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common; + +/** + * 服务对象的状态，通常需要start，shutdown + * + * @author shijia.wxr + */ +public enum ServiceState { + /** + * 服务对象刚刚创建，但是未启动 + */ + CREATE_JUST, + /** + * 服务启动成功 + */ + RUNNING, + /** + * 服务已经关闭 + */ + SHUTDOWN_ALREADY, + /** + * 服务启动失败 + */ + START_FAILED +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/ServiceThread.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/ServiceThread.java new file mode 100644 index 000000000..c856d2c56 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/ServiceThread.java @@ -0,0 +1,159 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.rocketmq.common.constant.LoggerName; + + +/** + * 后台服务线程基类 + * + * @author shijia.wxr + */ +public abstract class ServiceThread implements Runnable { + private static final Logger stlog = LoggerFactory.getLogger(LoggerName.CommonLoggerName); + // 执行线程 + protected final Thread thread; + // 线程回收时间，默认90S + private static final long JoinTime = 90 * 1000; + // 是否已经被Notify过 + protected volatile boolean hasNotified = false; + // 线程是否已经停止 + protected volatile boolean stoped = false; + + + public ServiceThread() { + this.thread = new Thread(this, this.getServiceName()); + } + + + public abstract String getServiceName(); + + + public void start() { + this.thread.start(); + } + + + public void shutdown() { + this.shutdown(false); + } + + + public void stop() { + this.stop(false); + } + + + public void makeStop() { + this.stoped = true; + stlog.info("makestop thread " + this.getServiceName()); + } + + + public void stop(final boolean interrupt) { + this.stoped = true; + stlog.info("stop thread " + this.getServiceName() + " interrupt " + interrupt); + synchronized (this) { + if (!this.hasNotified) { + this.hasNotified = true; + this.notify(); + } + } + + if (interrupt) { + this.thread.interrupt(); + } + } + + + public void shutdown(final boolean interrupt) { + this.stoped = true; + stlog.info("shutdown thread " + this.getServiceName() + " interrupt " + interrupt); + synchronized (this) { + if (!this.hasNotified) { + this.hasNotified = true; + this.notify(); + } + } + + try { + if (interrupt) { + this.thread.interrupt(); + } + + long beginTime = System.currentTimeMillis(); + if (!this.thread.isDaemon()) { + this.thread.join(this.getJointime()); + } + long eclipseTime = System.currentTimeMillis() - beginTime; + stlog.info("join thread " + this.getServiceName() + " eclipse time(ms) " + eclipseTime + " " + + this.getJointime()); + } + catch (InterruptedException e) { + e.printStackTrace(); + } + } + + + public void wakeup() { + synchronized (this) { + if (!this.hasNotified) { + this.hasNotified = true; + this.notify(); + } + } + } + + + protected void waitForRunning(long interval) { + synchronized (this) { + if (this.hasNotified) { + this.hasNotified = false; + this.onWaitEnd(); + return; + } + + try { + this.wait(interval); + } + catch (InterruptedException e) { + e.printStackTrace(); + } + finally { + this.hasNotified = false; + this.onWaitEnd(); + } + } + } + + + protected void onWaitEnd() { + } + + + public boolean isStoped() { + return stoped; + } + + + public long getJointime() { + return JoinTime; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/SystemClock.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/SystemClock.java new file mode 100644 index 000000000..c1cc94df3 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/SystemClock.java @@ -0,0 +1,72 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common; + +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + + +/** + * 后台定时更新时钟，JVM退出时，线程自动回收 + * + * @author vintage.wang@gmail.com shijia.wxr@taobao.com + * @see + * https://github.com/zhongl/jtoolkit/blob/master/common/src/main/java/com + * /github/zhongl/jtoolkit/SystemClock.java + */ +public class SystemClock { + + private final long precision; + private final AtomicLong now; + + + public SystemClock(long precision) { + this.precision = precision; + now = new AtomicLong(System.currentTimeMillis()); + scheduleClockUpdating(); + } + + + private void scheduleClockUpdating() { + ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(new ThreadFactory() { + @Override + public Thread newThread(Runnable runnable) { + Thread thread = new Thread(runnable, "System Clock"); + thread.setDaemon(true); + return thread; + } + }); + scheduler.scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + now.set(System.currentTimeMillis()); + } + }, precision, precision, TimeUnit.MILLISECONDS); + } + + + public long now() { + return now.get(); + } + + + public long precision() { + return precision; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/ThreadFactoryImpl.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/ThreadFactoryImpl.java new file mode 100644 index 000000000..5086ba2ab --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/ThreadFactoryImpl.java @@ -0,0 +1,22 @@ +package com.alibaba.rocketmq.common; + +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.atomic.AtomicLong; + + +public class ThreadFactoryImpl implements ThreadFactory { + private final AtomicLong threadIndex = new AtomicLong(0); + private final String threadNamePrefix; + + + public ThreadFactoryImpl(final String threadNamePrefix) { + this.threadNamePrefix = threadNamePrefix; + } + + + @Override + public Thread newThread(Runnable r) { + return new Thread(r, threadNamePrefix + this.threadIndex.incrementAndGet()); + + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/TopicConfig.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/TopicConfig.java new file mode 100644 index 000000000..280de6df9 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/TopicConfig.java @@ -0,0 +1,195 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common; + +import com.alibaba.rocketmq.common.constant.PermName; + + +/** + * Topic配置 + * + * @author shijia.wxr + */ +public class TopicConfig { + public static int DefaultReadQueueNums = 16; + public static int DefaultWriteQueueNums = 16; + + private static final String SEPARATOR = " "; + + private String topicName; + private int readQueueNums = DefaultReadQueueNums; + private int writeQueueNums = DefaultWriteQueueNums; + private int perm = PermName.PERM_READ | PermName.PERM_WRITE; + private TopicFilterType topicFilterType = TopicFilterType.SINGLE_TAG; + private int topicSysFlag = 0; + private boolean order = false; + + + public TopicConfig() { + } + + + public TopicConfig(String topicName) { + this.topicName = topicName; + } + + + public TopicConfig(String topicName, int readQueueNums, int writeQueueNums, int perm) { + this.topicName = topicName; + this.readQueueNums = readQueueNums; + this.writeQueueNums = writeQueueNums; + this.perm = perm; + } + + + public String encode() { + StringBuilder sb = new StringBuilder(); + + // 1 + sb.append(this.topicName); + sb.append(SEPARATOR); + + // 2 + sb.append(this.readQueueNums); + sb.append(SEPARATOR); + + // 3 + sb.append(this.writeQueueNums); + sb.append(SEPARATOR); + + // 4 + sb.append(this.perm); + sb.append(SEPARATOR); + + // 5 + sb.append(this.topicFilterType); + + return sb.toString(); + } + + + public boolean decode(final String in) { + String[] strs = in.split(SEPARATOR); + if (strs != null && strs.length == 5) { + this.topicName = strs[0]; + + this.readQueueNums = Integer.parseInt(strs[1]); + + this.writeQueueNums = Integer.parseInt(strs[2]); + + this.perm = Integer.parseInt(strs[3]); + + this.topicFilterType = TopicFilterType.valueOf(strs[4]); + + return true; + } + + return false; + } + + + public String getTopicName() { + return topicName; + } + + + public void setTopicName(String topicName) { + this.topicName = topicName; + } + + + public int getReadQueueNums() { + return readQueueNums; + } + + + public void setReadQueueNums(int readQueueNums) { + this.readQueueNums = readQueueNums; + } + + + public int getWriteQueueNums() { + return writeQueueNums; + } + + + public void setWriteQueueNums(int writeQueueNums) { + this.writeQueueNums = writeQueueNums; + } + + + public int getPerm() { + return perm; + } + + + public void setPerm(int perm) { + this.perm = perm; + } + + + public TopicFilterType getTopicFilterType() { + return topicFilterType; + } + + + public void setTopicFilterType(TopicFilterType topicFilterType) { + this.topicFilterType = topicFilterType; + } + + + public int getTopicSysFlag() { + return topicSysFlag; + } + + + public void setTopicSysFlag(int topicSysFlag) { + this.topicSysFlag = topicSysFlag; + } + + + public boolean isOrder() { + return order; + } + + + public void setOrder(boolean isOrder) { + this.order = isOrder; + } + + + @Override + public boolean equals(Object obj) { + TopicConfig other = (TopicConfig) obj; + if (other != null) { + return this.topicName.equals(other.topicName) && this.readQueueNums == other.readQueueNums + && this.writeQueueNums == other.writeQueueNums && this.perm == other.perm + && this.topicFilterType == other.topicFilterType + && this.topicSysFlag == other.topicSysFlag && this.order == other.order; + } + + return false; + } + + + @Override + public String toString() { + return "TopicConfig [topicName=" + topicName + ", readQueueNums=" + readQueueNums + + ", writeQueueNums=" + writeQueueNums + ", perm=" + PermName.perm2String(perm) + + ", topicFilterType=" + topicFilterType + ", topicSysFlag=" + topicSysFlag + ", order=" + + order + "]"; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/TopicFilterType.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/TopicFilterType.java new file mode 100644 index 000000000..7adbb5849 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/TopicFilterType.java @@ -0,0 +1,34 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common; + +/** + * Topic过滤方式，默认为单TAG过滤 + * + * @author shijia.wxr + */ +public enum TopicFilterType { + /** + * 每个消息只能有一个Tag + */ + SINGLE_TAG, + /** + * 每个消息可以有多个Tag（暂时不支持，后续视情况支持）
+ * 为什么暂时不支持？
+ * 此功能可能会对用户造成困扰，且方案并不完美，所以暂不支持 + */ + MULTI_TAG +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/UtilAll.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/UtilAll.java new file mode 100644 index 000000000..ae3698653 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/UtilAll.java @@ -0,0 +1,466 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.lang.management.ManagementFactory; +import java.lang.management.RuntimeMXBean; +import java.text.NumberFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Calendar; +import java.util.Date; +import java.util.Iterator; +import java.util.Map; +import java.util.zip.CRC32; +import java.util.zip.DeflaterOutputStream; +import java.util.zip.InflaterInputStream; + +import com.alibaba.rocketmq.remoting.common.RemotingHelper; + + +/** + * 各种方法大杂烩 + * + * @author shijia.wxr + */ +public class UtilAll { + public static final String yyyy_MM_dd_HH_mm_ss = "yyyy-MM-dd HH:mm:ss"; + public static final String yyyy_MM_dd_HH_mm_ss_SSS = "yyyy-MM-dd#HH:mm:ss:SSS"; + public static final String yyyyMMddHHmmss = "yyyyMMddHHmmss"; + + + public static int getPid() { + RuntimeMXBean runtime = ManagementFactory.getRuntimeMXBean(); + String name = runtime.getName(); // format: "pid@hostname" + try { + return Integer.parseInt(name.substring(0, name.indexOf('@'))); + } + catch (Exception e) { + return -1; + } + } + + + public static String currentStackTrace() { + StringBuilder sb = new StringBuilder(); + StackTraceElement[] stackTrace = Thread.currentThread().getStackTrace(); + for (StackTraceElement ste : stackTrace) { + sb.append("\n\t"); + sb.append(ste.toString()); + } + + return sb.toString(); + } + + + /** + * 将offset转化成字符串形式
+ * 左补零对齐至20位 + */ + public static String offset2FileName(final long offset) { + final NumberFormat nf = NumberFormat.getInstance(); + nf.setMinimumIntegerDigits(20); + nf.setMaximumFractionDigits(0); + nf.setGroupingUsed(false); + return nf.format(offset); + } + + + /** + * 计算耗时操作，单位ms + */ + public static long computeEclipseTimeMilliseconds(final long beginTime) { + return (System.currentTimeMillis() - beginTime); + } + + + public static boolean isItTimeToDo(final String when) { + String[] whiles = when.split(";"); + if (whiles != null && whiles.length > 0) { + Calendar now = Calendar.getInstance(); + for (String w : whiles) { + int nowHour = Integer.parseInt(w); + if (nowHour == now.get(Calendar.HOUR_OF_DAY)) { + return true; + } + } + } + + return false; + } + + + public static String timeMillisToHumanString() { + return timeMillisToHumanString(System.currentTimeMillis()); + } + + + public static String timeMillisToHumanString(final long t) { + Calendar cal = Calendar.getInstance(); + cal.setTimeInMillis(t); + return String.format("%04d%02d%02d%02d%02d%02d%03d", cal.get(Calendar.YEAR), + cal.get(Calendar.MONTH) + 1, cal.get(Calendar.DAY_OF_MONTH), cal.get(Calendar.HOUR_OF_DAY), + cal.get(Calendar.MINUTE), cal.get(Calendar.SECOND), cal.get(Calendar.MILLISECOND)); + } + + + public static long computNextMorningTimeMillis() { + Calendar cal = Calendar.getInstance(); + cal.setTimeInMillis(System.currentTimeMillis()); + cal.add(Calendar.DAY_OF_MONTH, 1); + cal.set(Calendar.HOUR_OF_DAY, 0); + cal.set(Calendar.MINUTE, 0); + cal.set(Calendar.SECOND, 0); + cal.set(Calendar.MILLISECOND, 0); + + return cal.getTimeInMillis(); + } + + + public static long computNextMinutesTimeMillis() { + Calendar cal = Calendar.getInstance(); + cal.setTimeInMillis(System.currentTimeMillis()); + cal.add(Calendar.DAY_OF_MONTH, 0); + cal.add(Calendar.HOUR_OF_DAY, 0); + cal.add(Calendar.MINUTE, 1); + cal.set(Calendar.SECOND, 0); + cal.set(Calendar.MILLISECOND, 0); + + return cal.getTimeInMillis(); + } + + + public static long computNextHourTimeMillis() { + Calendar cal = Calendar.getInstance(); + cal.setTimeInMillis(System.currentTimeMillis()); + cal.add(Calendar.DAY_OF_MONTH, 0); + cal.add(Calendar.HOUR_OF_DAY, 1); + cal.set(Calendar.MINUTE, 0); + cal.set(Calendar.SECOND, 0); + cal.set(Calendar.MILLISECOND, 0); + + return cal.getTimeInMillis(); + } + + + public static long computNextHalfHourTimeMillis() { + Calendar cal = Calendar.getInstance(); + cal.setTimeInMillis(System.currentTimeMillis()); + cal.add(Calendar.DAY_OF_MONTH, 0); + cal.add(Calendar.HOUR_OF_DAY, 1); + cal.set(Calendar.MINUTE, 30); + cal.set(Calendar.SECOND, 0); + cal.set(Calendar.MILLISECOND, 0); + + return cal.getTimeInMillis(); + } + + + public static String timeMillisToHumanString2(final long t) { + Calendar cal = Calendar.getInstance(); + cal.setTimeInMillis(t); + return String.format("%04d-%02d-%02d %02d:%02d:%02d,%03d",// + cal.get(Calendar.YEAR),// + cal.get(Calendar.MONTH) + 1,// + cal.get(Calendar.DAY_OF_MONTH),// + cal.get(Calendar.HOUR_OF_DAY),// + cal.get(Calendar.MINUTE),// + cal.get(Calendar.SECOND),// + cal.get(Calendar.MILLISECOND)); + } + + + /** + * 返回日期时间格式，精度到秒
+ * 格式如下：2013122305190000 + * + * @param t + * @return + */ + public static String timeMillisToHumanString3(final long t) { + Calendar cal = Calendar.getInstance(); + cal.setTimeInMillis(t); + return String.format("%04d%02d%02d%02d%02d%02d",// + cal.get(Calendar.YEAR),// + cal.get(Calendar.MONTH) + 1,// + cal.get(Calendar.DAY_OF_MONTH),// + cal.get(Calendar.HOUR_OF_DAY),// + cal.get(Calendar.MINUTE),// + cal.get(Calendar.SECOND)); + } + + + /** + * 获取磁盘分区空间使用率 + */ + public static double getDiskPartitionSpaceUsedPercent(final String path) { + if (null == path || path.isEmpty()) + return -1; + + try { + File file = new File(path); + if (!file.exists()) { + boolean result = file.mkdirs(); + if (!result) { + // TODO + } + } + + long totalSpace = file.getTotalSpace(); + long freeSpace = file.getFreeSpace(); + long usedSpace = totalSpace - freeSpace; + if (totalSpace > 0) { + return usedSpace / (double) totalSpace; + } + } + catch (Exception e) { + return -1; + } + + return -1; + } + + + public static final int crc32(byte[] array) { + if (array != null) { + return crc32(array, 0, array.length); + } + + return 0; + } + + + public static final int crc32(byte[] array, int offset, int length) { + CRC32 crc32 = new CRC32(); + crc32.update(array, offset, length); + return (int) (crc32.getValue() & 0x7FFFFFFF); + } + + + /** + * 字节数组转化成16进制形式 + */ + public static String bytes2string(byte[] src) { + StringBuilder sb = new StringBuilder(); + if (src == null || src.length <= 0) { + return null; + } + for (int i = 0; i < src.length; i++) { + int v = src[i] & 0xFF; + String hv = Integer.toHexString(v); + if (hv.length() < 2) { + sb.append(0); + } + sb.append(hv.toUpperCase()); + } + return sb.toString(); + } + + + /** + * 16进制字符串转化成字节数组 + */ + public static byte[] string2bytes(String hexString) { + if (hexString == null || hexString.equals("")) { + return null; + } + hexString = hexString.toUpperCase(); + int length = hexString.length() / 2; + char[] hexChars = hexString.toCharArray(); + byte[] d = new byte[length]; + for (int i = 0; i < length; i++) { + int pos = i * 2; + d[i] = (byte) (charToByte(hexChars[pos]) << 4 | charToByte(hexChars[pos + 1])); + } + return d; + } + + + private static byte charToByte(char c) { + return (byte) "0123456789ABCDEF".indexOf(c); + } + + + public static byte[] uncompress(final byte[] src) throws IOException { + byte[] result = src; + byte[] uncompressData = new byte[src.length]; + ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(src); + InflaterInputStream inflaterInputStream = new InflaterInputStream(byteArrayInputStream); + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(src.length); + + try { + while (true) { + int len = inflaterInputStream.read(uncompressData, 0, uncompressData.length); + if (len <= 0) { + break; + } + byteArrayOutputStream.write(uncompressData, 0, len); + } + byteArrayOutputStream.flush(); + result = byteArrayOutputStream.toByteArray(); + } + catch (IOException e) { + throw e; + } + finally { + try { + byteArrayInputStream.close(); + } + catch (IOException e) { + } + try { + inflaterInputStream.close(); + } + catch (IOException e) { + } + try { + byteArrayOutputStream.close(); + } + catch (IOException e) { + } + } + + return result; + } + + + public static byte[] compress(final byte[] src, final int level) throws IOException { + byte[] result = src; + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(src.length); + java.util.zip.Deflater deflater = new java.util.zip.Deflater(level); + DeflaterOutputStream deflaterOutputStream = new DeflaterOutputStream(byteArrayOutputStream, deflater); + try { + deflaterOutputStream.write(src); + deflaterOutputStream.finish(); + deflaterOutputStream.close(); + result = byteArrayOutputStream.toByteArray(); + } + catch (IOException e) { + deflater.end(); + throw e; + } + finally { + try { + byteArrayOutputStream.close(); + } + catch (IOException e) { + } + + deflater.end(); + } + + return result; + } + + + public static int asInt(String str, int defaultValue) { + try { + return Integer.parseInt(str); + } + catch (Exception e) { + return defaultValue; + } + } + + + public static long asLong(String str, long defaultValue) { + try { + return Long.parseLong(str); + } + catch (Exception e) { + return defaultValue; + } + } + + + public static String formatDate(Date date, String pattern) { + SimpleDateFormat df = new SimpleDateFormat(pattern); + return df.format(date); + } + + + public static Date parseDate(String date, String pattern) { + SimpleDateFormat df = new SimpleDateFormat(pattern); + try { + return df.parse(date); + } + catch (ParseException e) { + return null; + } + } + + + public static String responseCode2String(final int code) { + return Integer.toString(code); + } + + + public static String frontStringAtLeast(final String str, final int size) { + if (str != null) { + if (str.length() > size) { + return str.substring(0, size); + } + } + + return str; + } + + + public static boolean isBlank(String str) { + int strLen; + if (str == null || (strLen = str.length()) == 0) { + return true; + } + for (int i = 0; i < strLen; i++) { + if ((Character.isWhitespace(str.charAt(i)) == false)) { + return false; + } + } + return true; + } + + + public static String jstack() { + StringBuilder result = new StringBuilder(); + try { + Map map = Thread.getAllStackTraces(); + Iterator> ite = map.entrySet().iterator(); + while (ite.hasNext()) { + Map.Entry entry = ite.next(); + StackTraceElement[] elements = entry.getValue(); + Thread thread = entry.getKey(); + if (elements != null && elements.length > 0) { + String threadName = entry.getKey().getName(); + result.append(String.format("%-40sTID: %d STATE: %s\n", threadName, thread.getId(), + thread.getState())); + for (StackTraceElement el : elements) { + result.append(String.format("%-40s%s\n", threadName, el.toString())); + } + result.append("\n"); + } + } + } + catch (Throwable e) { + result.append(RemotingHelper.exceptionSimpleDesc(e)); + } + + return result.toString(); + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/admin/ConsumeStats.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/admin/ConsumeStats.java new file mode 100644 index 000000000..d952f213e --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/admin/ConsumeStats.java @@ -0,0 +1,69 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common.admin; + +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map.Entry; + +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; + + +/** + * Consumer消费进度 + * + * @author shijia.wxr + * @since 2013-7-14 + */ +public class ConsumeStats extends RemotingSerializable { + private HashMap offsetTable = new HashMap(); + private long consumeTps = 0; + + + public long computeTotalDiff() { + long diffTotal = 0L; + + Iterator> it = this.offsetTable.entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + long diff = next.getValue().getBrokerOffset() - next.getValue().getConsumerOffset(); + diffTotal += diff; + } + + return diffTotal; + } + + + public HashMap getOffsetTable() { + return offsetTable; + } + + + public void setOffsetTable(HashMap offsetTable) { + this.offsetTable = offsetTable; + } + + + public long getConsumeTps() { + return consumeTps; + } + + + public void setConsumeTps(long consumeTps) { + this.consumeTps = consumeTps; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/admin/OffsetWrapper.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/admin/OffsetWrapper.java new file mode 100644 index 000000000..65a95efa0 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/admin/OffsetWrapper.java @@ -0,0 +1,59 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common.admin; + +/** + * Offset包装类，含Broker、Consumer + * + * @author shijia.wxr + * @since 2013-7-14 + */ +public class OffsetWrapper { + private long brokerOffset; + private long consumerOffset; + // 消费的最后一条消息对应的时间戳 + private long lastTimestamp; + + + public long getBrokerOffset() { + return brokerOffset; + } + + + public void setBrokerOffset(long brokerOffset) { + this.brokerOffset = brokerOffset; + } + + + public long getConsumerOffset() { + return consumerOffset; + } + + + public void setConsumerOffset(long consumerOffset) { + this.consumerOffset = consumerOffset; + } + + + public long getLastTimestamp() { + return lastTimestamp; + } + + + public void setLastTimestamp(long lastTimestamp) { + this.lastTimestamp = lastTimestamp; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/admin/RollbackStats.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/admin/RollbackStats.java new file mode 100644 index 000000000..caeba7096 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/admin/RollbackStats.java @@ -0,0 +1,76 @@ +package com.alibaba.rocketmq.common.admin; + +/** + * 按时间回溯消费进度 + * + * @author: manhong.yqd + * @since: 13-9-12 + */ +public class RollbackStats { + private String brokerName; + private long queueId; + private long brokerOffset; + private long consumerOffset; + private long timestampOffset; + private long rollbackOffset; + + + public String getBrokerName() { + return brokerName; + } + + + public void setBrokerName(String brokerName) { + this.brokerName = brokerName; + } + + + public long getQueueId() { + return queueId; + } + + + public void setQueueId(long queueId) { + this.queueId = queueId; + } + + + public long getBrokerOffset() { + return brokerOffset; + } + + + public void setBrokerOffset(long brokerOffset) { + this.brokerOffset = brokerOffset; + } + + + public long getConsumerOffset() { + return consumerOffset; + } + + + public void setConsumerOffset(long consumerOffset) { + this.consumerOffset = consumerOffset; + } + + + public long getTimestampOffset() { + return timestampOffset; + } + + + public void setTimestampOffset(long timestampOffset) { + this.timestampOffset = timestampOffset; + } + + + public long getRollbackOffset() { + return rollbackOffset; + } + + + public void setRollbackOffset(long rollbackOffset) { + this.rollbackOffset = rollbackOffset; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/admin/TopicOffset.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/admin/TopicOffset.java new file mode 100644 index 000000000..02689db78 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/admin/TopicOffset.java @@ -0,0 +1,58 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common.admin; + +/** + * Topic 统计信息信息 + * + * @author shijia.wxr + * @since 2013-7-14 + */ +public class TopicOffset { + private long minOffset; + private long maxOffset; + private long lastUpdateTimestamp; + + + public long getMinOffset() { + return minOffset; + } + + + public void setMinOffset(long minOffset) { + this.minOffset = minOffset; + } + + + public long getMaxOffset() { + return maxOffset; + } + + + public void setMaxOffset(long maxOffset) { + this.maxOffset = maxOffset; + } + + + public long getLastUpdateTimestamp() { + return lastUpdateTimestamp; + } + + + public void setLastUpdateTimestamp(long lastUpdateTimestamp) { + this.lastUpdateTimestamp = lastUpdateTimestamp; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/admin/TopicStatsTable.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/admin/TopicStatsTable.java new file mode 100644 index 000000000..1e8217d68 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/admin/TopicStatsTable.java @@ -0,0 +1,42 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common.admin; + +import java.util.HashMap; + +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; + + +/** + * Topic所有队列的Offset + * + * @author shijia.wxr + * @since 2013-7-14 + */ +public class TopicStatsTable extends RemotingSerializable { + private HashMap offsetTable = new HashMap(); + + + public HashMap getOffsetTable() { + return offsetTable; + } + + + public void setOffsetTable(HashMap offsetTable) { + this.offsetTable = offsetTable; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/annotation/ImportantField.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/annotation/ImportantField.java new file mode 100644 index 000000000..525cb4bf7 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/annotation/ImportantField.java @@ -0,0 +1,29 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common.annotation; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + + +@Documented +@Retention(RetentionPolicy.RUNTIME) +@Target({ ElementType.FIELD, ElementType.METHOD, ElementType.PARAMETER, ElementType.LOCAL_VARIABLE }) +public @interface ImportantField { +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/constant/LoggerName.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/constant/LoggerName.java new file mode 100644 index 000000000..d54eac1a6 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/constant/LoggerName.java @@ -0,0 +1,34 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common.constant; + +/** + * @author shijia.wxr + */ +public class LoggerName { + public static final String FiltersrvLoggerName = "RocketmqFiltersrv"; + public static final String NamesrvLoggerName = "RocketmqNamesrv"; + public static final String BrokerLoggerName = "RocketmqBroker"; + public static final String ClientLoggerName = "RocketmqClient"; + public static final String ToolsLoggerName = "RocketmqTools"; + public static final String CommonLoggerName = "RocketmqCommon"; + public static final String StoreLoggerName = "RocketmqStore"; + public static final String StoreErrorLoggerName = "RocketmqStoreError"; + public static final String TransactionLoggerName = "RocketmqTransaction"; + public static final String RebalanceLockLoggerName = "RocketmqRebalanceLock"; + public static final String RocketmqStatsLoggerName = "RocketmqStats"; + public static final String RocketmqAuthorizeLoggerName = "RocketmqAuthorize"; +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/constant/PermName.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/constant/PermName.java new file mode 100644 index 000000000..a43435e4f --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/constant/PermName.java @@ -0,0 +1,59 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common.constant; + +/** + * @author shijia.wxr + */ +public class PermName { + public static final int PERM_PRIORITY = 0x1 << 3; + public static final int PERM_READ = 0x1 << 2; + public static final int PERM_WRITE = 0x1 << 1; + public static final int PERM_INHERIT = 0x1 << 0; + + + public static boolean isReadable(final int perm) { + return (perm & PERM_READ) == PERM_READ; + } + + + public static boolean isWriteable(final int perm) { + return (perm & PERM_WRITE) == PERM_WRITE; + } + + + public static boolean isInherited(final int perm) { + return (perm & PERM_INHERIT) == PERM_INHERIT; + } + + + public static String perm2String(final int perm) { + final StringBuffer sb = new StringBuffer("---"); + if (isReadable(perm)) { + sb.replace(0, 1, "R"); + } + + if (isWriteable(perm)) { + sb.replace(1, 2, "W"); + } + + if (isInherited(perm)) { + sb.replace(2, 3, "X"); + } + + return sb.toString(); + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/consumer/ConsumeFromWhere.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/consumer/ConsumeFromWhere.java new file mode 100644 index 000000000..24c13386b --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/consumer/ConsumeFromWhere.java @@ -0,0 +1,47 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common.consumer; + +/** + * Consumer从哪里开始消费
+ * + * @author shijia.wxr + */ +public enum ConsumeFromWhere { + /** + * 一个新的订阅组第一次启动从队列的最后位置开始消费
+ * 后续再启动接着上次消费的进度开始消费 + */ + CONSUME_FROM_LAST_OFFSET, + + @Deprecated + CONSUME_FROM_LAST_OFFSET_AND_FROM_MIN_WHEN_BOOT_FIRST, + @Deprecated + CONSUME_FROM_MIN_OFFSET, + @Deprecated + CONSUME_FROM_MAX_OFFSET, + /** + * 一个新的订阅组第一次启动从队列的最前位置开始消费
+ * 后续再启动接着上次消费的进度开始消费 + */ + CONSUME_FROM_FIRST_OFFSET, + /** + * 一个新的订阅组第一次启动从指定时间点开始消费
+ * 后续再启动接着上次消费的进度开始消费
+ * 时间点设置参见DefaultMQPushConsumer.consumeTimestamp参数 + */ + CONSUME_FROM_TIMESTAMP, +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/filter/FilterAPI.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/filter/FilterAPI.java new file mode 100644 index 000000000..e7df8c8e0 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/filter/FilterAPI.java @@ -0,0 +1,107 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common.filter; + +import java.net.URL; + +import com.alibaba.rocketmq.common.protocol.heartbeat.SubscriptionData; + + +/** + * @author shijia.wxr + * @since 2013-6-15 + */ +public class FilterAPI { + public static String simpleClassName(final String className) { + String simple = className; + int index = className.lastIndexOf("."); + if (index >= 0) { + simple = className.substring(index + 1); + } + + return simple; + } + + + public static URL classFile(final String className) { + final String javaSource = simpleClassName(className) + ".java"; + URL url = FilterAPI.class.getClassLoader().getResource(javaSource); + return url; + } + + + public static boolean isFilterClassMode(final String subString) { + try { + if (subString.contains(".")) { + // Class loadClass = + // FilterAPI.class.getClassLoader().loadClass(subString); + // Class[] interfaces = loadClass.getInterfaces(); + // for (int i = 0; i < interfaces.length; i++) { + // if + // (interfaces[i].getCanonicalName().equals(MessageFilter.class.getCanonicalName())) + // { + // return true; + // } + // } + return true; + } + } + catch (Exception e) { + } + + return false; + } + + + public static SubscriptionData buildSubscriptionData(final String consumerGroup, String topic, + String subString) throws Exception { + SubscriptionData subscriptionData = new SubscriptionData(); + subscriptionData.setTopic(topic); + subscriptionData.setSubString(subString); + + if (null == subString || subString.equals(SubscriptionData.SUB_ALL) || subString.length() == 0) { + subscriptionData.setSubString(SubscriptionData.SUB_ALL); + } + // eg: com.taobao.abc.FilterClassName + else if (isFilterClassMode(subString)) { + // if (null == classFile(subString)) { + // throw new + // Exception(String.format("The Filter Java Class Source[%s] not exist in class path", + // subString + ".java")); + // } + subscriptionData.setClassFilterMode(true); + } + else { + String[] tags = subString.split("\\|\\|"); + if (tags != null && tags.length > 0) { + for (String tag : tags) { + if (tag.length() > 0) { + String trimString = tag.trim(); + if (trimString.length() > 0) { + subscriptionData.getTagsSet().add(trimString); + subscriptionData.getCodeSet().add(trimString.hashCode()); + } + } + } + } + else { + throw new Exception("subString split error"); + } + } + + return subscriptionData; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/filter/MessageFilter.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/filter/MessageFilter.java new file mode 100644 index 000000000..3dfd5f314 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/filter/MessageFilter.java @@ -0,0 +1,18 @@ +package com.alibaba.rocketmq.common.filter; + +import com.alibaba.rocketmq.common.message.MessageExt; + + +/** + * 服务端消息过滤接口，Consumer实现这个接口后，Consumer客户端会注册这段Java程序到Broker，由Broker来编译并执行， + * 以达到服务器消息过滤的目的 + */ +public interface MessageFilter { + /** + * 过滤消息 + * + * @param msg + * @return 是否可以被Consumer消费 + */ + public boolean match(final MessageExt msg); +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/filter/impl/Op.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/filter/impl/Op.java new file mode 100644 index 000000000..5bca13f4d --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/filter/impl/Op.java @@ -0,0 +1,24 @@ +package com.alibaba.rocketmq.common.filter.impl; + +/** + * @auther lansheng.zj@taobao.com + */ +public abstract class Op { + + private String symbol; + + + protected Op(String symbol) { + this.symbol = symbol; + } + + + public String getSymbol() { + return symbol; + } + + + public String toString() { + return symbol; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/filter/impl/Operand.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/filter/impl/Operand.java new file mode 100644 index 000000000..5c5a4c4d2 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/filter/impl/Operand.java @@ -0,0 +1,12 @@ +package com.alibaba.rocketmq.common.filter.impl; + +/** + * @auther lansheng.zj@taobao.com + */ +public class Operand extends Op { + + public Operand(String symbol) { + super(symbol); + } + +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/filter/impl/Operator.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/filter/impl/Operator.java new file mode 100644 index 000000000..615554628 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/filter/impl/Operator.java @@ -0,0 +1,62 @@ +package com.alibaba.rocketmq.common.filter.impl; + +/** + * @auther lansheng.zj@taobao.com + */ +public class Operator extends Op { + + public static final Operator LEFTPARENTHESIS = new Operator("(", 30, false); + public static final Operator RIGHTPARENTHESIS = new Operator(")", 30, false); + public static final Operator AND = new Operator("&&", 20, true); + public static final Operator OR = new Operator("||", 15, true); + + private int priority; + private boolean compareable; + + + private Operator(String symbol, int priority, boolean compareable) { + super(symbol); + this.priority = priority; + this.compareable = compareable; + } + + + public int getPriority() { + return priority; + } + + + public boolean isCompareable() { + return compareable; + } + + + // -1 小于; 0 等于; 1大于 + public int compare(Operator operator) { + if (this.priority > operator.priority) + return 1; + else if (this.priority == operator.priority) + return 0; + else + return -1; + } + + + public boolean isSpecifiedOp(String operator) { + return this.getSymbol().equals(operator); + } + + + public static Operator createOperator(String operator) { + if (LEFTPARENTHESIS.getSymbol().equals(operator)) + return LEFTPARENTHESIS; + else if (RIGHTPARENTHESIS.getSymbol().equals(operator)) + return RIGHTPARENTHESIS; + else if (AND.getSymbol().equals(operator)) + return AND; + else if (OR.getSymbol().equals(operator)) + return OR; + else + throw new IllegalArgumentException("unsupport operator " + operator); + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/filter/impl/PolishExpr.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/filter/impl/PolishExpr.java new file mode 100644 index 000000000..0d432f3a7 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/filter/impl/PolishExpr.java @@ -0,0 +1,198 @@ +package com.alibaba.rocketmq.common.filter.impl; + +import static com.alibaba.rocketmq.common.filter.impl.Operator.LEFTPARENTHESIS; +import static com.alibaba.rocketmq.common.filter.impl.Operator.RIGHTPARENTHESIS; +import static com.alibaba.rocketmq.common.filter.impl.Operator.createOperator; + +import java.util.ArrayList; +import java.util.List; +import java.util.Stack; + + +/** + * @auther lansheng.zj@taobao.com + */ +public class PolishExpr { + + /** + * 拆分单词 + * + * @param expression + * @return + * @throws Exception + */ + private static List participle(String expression) { + List segments = new ArrayList(); + + int size = expression.length(); + int wordStartIndex = -1; + int wordLen = 0; + Type preType = Type.NULL; + + for (int i = 0; i < size; i++) { + int chValue = (int) expression.charAt(i); + + if ((97 <= chValue && chValue <= 122) || (65 <= chValue && chValue <= 90) + || (49 <= chValue && chValue <= 57) || 95 == chValue) { + // 操作数 + + if (Type.OPERATOR == preType || Type.SEPAERATOR == preType || Type.NULL == preType + || Type.PARENTHESIS == preType) { + if (Type.OPERATOR == preType) { + segments.add(createOperator(expression.substring(wordStartIndex, wordStartIndex + + wordLen))); + } + wordStartIndex = i; + wordLen = 0; + } + preType = Type.OPERAND; + wordLen++; + } + else if (40 == chValue || 41 == chValue) { + // 括号 + + if (Type.OPERATOR == preType) { + segments.add(createOperator(expression + .substring(wordStartIndex, wordStartIndex + wordLen))); + wordStartIndex = -1; + wordLen = 0; + } + else if (Type.OPERAND == preType) { + segments.add(new Operand(expression.substring(wordStartIndex, wordStartIndex + wordLen))); + wordStartIndex = -1; + wordLen = 0; + } + + preType = Type.PARENTHESIS; + segments.add(createOperator((char) chValue + "")); + } + else if (38 == chValue || 124 == chValue) { + // 操作符 + if (Type.OPERAND == preType || Type.SEPAERATOR == preType || Type.PARENTHESIS == preType) { + if (Type.OPERAND == preType) { + segments.add(new Operand(expression.substring(wordStartIndex, wordStartIndex + + wordLen))); + } + wordStartIndex = i; + wordLen = 0; + } + preType = Type.OPERATOR; + wordLen++; + } + else if (32 == chValue || 9 == chValue) { + // 单词分隔符 + + if (Type.OPERATOR == preType) { + segments.add(createOperator(expression + .substring(wordStartIndex, wordStartIndex + wordLen))); + wordStartIndex = -1; + wordLen = 0; + } + else if (Type.OPERAND == preType) { + segments.add(new Operand(expression.substring(wordStartIndex, wordStartIndex + wordLen))); + wordStartIndex = -1; + wordLen = 0; + } + preType = Type.SEPAERATOR; + } + else { + // 非法字符 + throw new IllegalArgumentException("illegal expression, at index " + i + " " + (char) chValue); + } + + } + + if (wordLen > 0) { + segments.add(new Operand(expression.substring(wordStartIndex, wordStartIndex + wordLen))); + } + return segments; + } + + + /** + * 将中缀表达式转换成逆波兰表达式 + * + * @param expression + * @return + */ + public static List reversePolish(String expression) { + return reversePolish(participle(expression)); + } + + + /** + * 将中缀表达式转换成逆波兰表达式
+ * Shunting-yard algorithm
+ * http://en.wikipedia.org/wiki/Shunting_yard_algorithm + * + * @param tokens + * @return + */ + public static List reversePolish(List tokens) { + List segments = new ArrayList(); + Stack operatorStack = new Stack(); + + for (int i = 0; i < tokens.size(); i++) { + Op token = tokens.get(i); + if (isOperand(token)) { + // 操作数 + segments.add(token); + } + else if (isLeftParenthesis(token)) { + // 左括号 + operatorStack.push((Operator) token); + } + else if (isRightParenthesis(token)) { + // 右括号 + Operator opNew = null; + while (!operatorStack.empty() && LEFTPARENTHESIS != (opNew = operatorStack.pop())) { + segments.add(opNew); + } + if (null == opNew || LEFTPARENTHESIS != opNew) + throw new IllegalArgumentException("mismatched parentheses"); + } + else if (isOperator(token)) { + // 操作符,暂不考虑结合性(左结合,右结合),支持的操作符都是左结合的 + Operator opNew = (Operator) token; + if (!operatorStack.empty()) { + Operator opOld = operatorStack.peek(); + if (opOld.isCompareable() && opNew.compare(opOld) != 1) { + segments.add(operatorStack.pop()); + } + } + operatorStack.push(opNew); + } + else + throw new IllegalArgumentException("illegal token " + token); + } + + while (!operatorStack.empty()) { + Operator operator = operatorStack.pop(); + if (LEFTPARENTHESIS == operator || RIGHTPARENTHESIS == operator) + throw new IllegalArgumentException("mismatched parentheses " + operator); + segments.add(operator); + } + + return segments; + } + + + public static boolean isOperand(Op token) { + return token instanceof Operand; + } + + + public static boolean isOperator(Op token) { + return token instanceof Operator; + } + + + public static boolean isLeftParenthesis(Op token) { + return token instanceof Operator && LEFTPARENTHESIS == (Operator) token; + } + + + public static boolean isRightParenthesis(Op token) { + return token instanceof Operator && RIGHTPARENTHESIS == (Operator) token; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/filter/impl/Type.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/filter/impl/Type.java new file mode 100644 index 000000000..7c5636b0d --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/filter/impl/Type.java @@ -0,0 +1,12 @@ +package com.alibaba.rocketmq.common.filter.impl; + +/** + * @auther lansheng.zj@taobao.com + */ +public enum Type { + NULL, + OPERAND, + OPERATOR, + PARENTHESIS, + SEPAERATOR; +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/help/FAQUrl.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/help/FAQUrl.java new file mode 100644 index 000000000..004de6dcf --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/help/FAQUrl.java @@ -0,0 +1,105 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common.help; + +/** + * 记录一些问题对应的解决方案，减少答疑工作量 + * + * @author shijia.wxr + */ +public class FAQUrl { + // FAQ: Topic不存在如何解决 + public static final String APPLY_TOPIC_URL = // + "https://github.com/alibaba/RocketMQ/issues/55"; + + // FAQ: 同一台机器无法启动多个实例（在多个JVM进程中） + public static final String CLIENT_INSTACNCE_NAME_DUPLICATE_URL = // + "https://github.com/alibaba/RocketMQ/issues/56"; + + // FAQ: Name Server地址不存在 + public static final String NAME_SERVER_ADDR_NOT_EXIST_URL = // + "https://github.com/alibaba/RocketMQ/issues/57"; + + // FAQ: 启动Producer、Consumer失败，Group Name重复 + public static final String GROUP_NAME_DUPLICATE_URL = // + "https://github.com/alibaba/RocketMQ/issues/63"; + + // FAQ: 客户端对象参数校验合法性 + public static final String CLIENT_PARAMETER_CHECK_URL = // + "https://github.com/alibaba/RocketMQ/issues/73"; + + // FAQ: 订阅组不存在如何解决 + public static final String SUBSCRIPTION_GROUP_NOT_EXIST = // + "https://github.com/alibaba/RocketMQ/issues/75"; + + // FAQ: Producer、Consumer服务状态不正确 + public static final String CLIENT_SERVICE_NOT_OK = // + "https://github.com/alibaba/RocketMQ/issues/214"; + + // FAQ: No route info of this topic, TopicABC + public static final String NO_TOPIC_ROUTE_INFO = // + "https://github.com/alibaba/RocketMQ/issues/264"; + + // FAQ: 广播消费者启动加载json文件异常问题 + public static final String LOAD_JSON_EXCEPTION = // + "https://github.com/alibaba/RocketMQ/issues/293"; + + // FAQ: 同一个订阅组内不同Consumer实例订阅关系不同 + public static final String SAME_GROUP_DIFFERENT_TOPIC = // + "https://github.com/alibaba/RocketMQ/issues/332"; + + // FAQ: 主动订阅消息，获取队列列表报Topic不存在 + public static final String MQLIST_NOT_EXIST = // + "https://github.com/alibaba/RocketMQ/issues/336"; + + // + // FAQ: 未收录异常处理办法 + // + public static final String UNEXPECTED_EXCEPTION_URL = // + "https://github.com/alibaba/RocketMQ/issues/64"; + + private static final String TipStringBegin = "\nSee "; + private static final String TipStringEnd = " for further details."; + + + public static String suggestTodo(final String url) { + StringBuilder sb = new StringBuilder(); + sb.append(TipStringBegin); + sb.append(url); + sb.append(TipStringEnd); + return sb.toString(); + } + + + /** + * 对于没有未异常原因指定FAQ的情况，追加默认FAQ + */ + public static String attachDefaultURL(final String errorMessage) { + if (errorMessage != null) { + int index = errorMessage.indexOf(TipStringBegin); + if (-1 == index) { + StringBuilder sb = new StringBuilder(); + sb.append(errorMessage); + sb.append("\n"); + sb.append("For more information, please visit the url, "); + sb.append(UNEXPECTED_EXCEPTION_URL); + return sb.toString(); + } + } + + return errorMessage; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/hook/FilterCheckHook.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/hook/FilterCheckHook.java new file mode 100644 index 000000000..704516d23 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/hook/FilterCheckHook.java @@ -0,0 +1,17 @@ +package com.alibaba.rocketmq.common.hook; + +import java.nio.ByteBuffer; + + +/** + * 确认消息是否需要过滤 Hook + * + * @author manhong.yqd + * @since 2014-3-19 + */ +public interface FilterCheckHook { + public String hookName(); + + + public boolean isFilterMatched(final boolean isUnitMode, final ByteBuffer byteBuffer); +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/message/Message.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/message/Message.java new file mode 100644 index 000000000..7cf6f5962 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/message/Message.java @@ -0,0 +1,239 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common.message; + +import java.io.Serializable; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; + + +/** + * 消息，Producer与Consumer使用 + * + * @author shijia.wxr + * @since 2013-7-18 + */ +public class Message implements Serializable { + private static final long serialVersionUID = 8445773977080406428L; + + /** + * 消息主题 + */ + private String topic; + /** + * 消息标志，系统不做干预，完全由应用决定如何使用 + */ + private int flag; + /** + * 消息属性，都是系统属性，禁止应用设置 + */ + private Map properties; + /** + * 消息体 + */ + private byte[] body; + + + public Message() { + } + + + public Message(String topic, byte[] body) { + this(topic, "", "", 0, body, true); + } + + + public Message(String topic, String tags, byte[] body) { + this(topic, tags, "", 0, body, true); + } + + + public Message(String topic, String tags, String keys, byte[] body) { + this(topic, tags, keys, 0, body, true); + } + + + public Message(String topic, String tags, String keys, int flag, byte[] body, boolean waitStoreMsgOK) { + this.topic = topic; + this.flag = flag; + this.body = body; + + if (tags != null && tags.length() > 0) + this.setTags(tags); + + if (keys != null && keys.length() > 0) + this.setKeys(keys); + + this.setWaitStoreMsgOK(waitStoreMsgOK); + } + + + void clearProperty(final String name) { + if (null != this.properties) { + this.properties.remove(name); + } + } + + + void putProperty(final String name, final String value) { + if (null == this.properties) { + this.properties = new HashMap(); + } + + this.properties.put(name, value); + } + + + public void putUserProperty(final String name, final String value) { + if (MessageConst.systemKeySet.contains(name)) { + throw new RuntimeException(String.format( + "The Property<%s> is used by system, input another please", name)); + } + this.putProperty(name, value); + } + + + public String getUserProperty(final String name) { + return this.getProperty(name); + } + + + public String getProperty(final String name) { + if (null == this.properties) { + this.properties = new HashMap(); + } + + return this.properties.get(name); + } + + + public String getTopic() { + return topic; + } + + + public void setTopic(String topic) { + this.topic = topic; + } + + + public String getTags() { + return this.getProperty(MessageConst.PROPERTY_TAGS); + } + + + public void setTags(String tags) { + this.putProperty(MessageConst.PROPERTY_TAGS, tags); + } + + + public String getKeys() { + return this.getProperty(MessageConst.PROPERTY_KEYS); + } + + + public void setKeys(String keys) { + this.putProperty(MessageConst.PROPERTY_KEYS, keys); + } + + + public void setKeys(Collection keys) { + StringBuffer sb = new StringBuffer(); + for (String k : keys) { + sb.append(k); + sb.append(MessageConst.KEY_SEPARATOR); + } + + this.setKeys(sb.toString().trim()); + } + + + public int getDelayTimeLevel() { + String t = this.getProperty(MessageConst.PROPERTY_DELAY_TIME_LEVEL); + if (t != null) { + return Integer.parseInt(t); + } + + return 0; + } + + + public void setDelayTimeLevel(int level) { + this.putProperty(MessageConst.PROPERTY_DELAY_TIME_LEVEL, String.valueOf(level)); + } + + + public boolean isWaitStoreMsgOK() { + String result = this.getProperty(MessageConst.PROPERTY_WAIT_STORE_MSG_OK); + if (null == result) + return true; + + return Boolean.parseBoolean(result); + } + + + public void setWaitStoreMsgOK(boolean waitStoreMsgOK) { + this.putProperty(MessageConst.PROPERTY_WAIT_STORE_MSG_OK, Boolean.toString(waitStoreMsgOK)); + } + + + public int getFlag() { + return flag; + } + + + public void setFlag(int flag) { + this.flag = flag; + } + + + public byte[] getBody() { + return body; + } + + + public void setBody(byte[] body) { + this.body = body; + } + + + public Map getProperties() { + return properties; + } + + + void setProperties(Map properties) { + this.properties = properties; + } + + + public void setBuyerId(String buyerId) { + putProperty(MessageConst.PROPERTY_BUYER_ID, buyerId); + } + + + public String getBuyerId() { + return getProperty(MessageConst.PROPERTY_BUYER_ID); + } + + + @Override + public String toString() { + return "Message [topic=" + topic + ", flag=" + flag + ", properties=" + properties + ", body=" + + (body != null ? body.length : 0) + "]"; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/message/MessageAccessor.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/message/MessageAccessor.java new file mode 100644 index 000000000..d4374172e --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/message/MessageAccessor.java @@ -0,0 +1,71 @@ +package com.alibaba.rocketmq.common.message; + +import java.util.Map; + + +public class MessageAccessor { + + public static void putProperty(final Message msg, final String name, final String value) { + msg.putProperty(name, value); + } + + + public static void clearProperty(final Message msg, final String name) { + msg.clearProperty(name); + } + + + public static void setProperties(final Message msg, Map properties) { + msg.setProperties(properties); + } + + + public static void setTransferFlag(final Message msg, String unit) { + putProperty(msg, MessageConst.PROPERTY_TRANSFER_FLAG, unit); + } + + + public static String getTransferFlag(final Message msg) { + return msg.getProperty(MessageConst.PROPERTY_TRANSFER_FLAG); + } + + + public static void setCorrectionFlag(final Message msg, String unit) { + putProperty(msg, MessageConst.PROPERTY_CORRECTION_FLAG, unit); + } + + + public static String getCorrectionFlag(final Message msg) { + return msg.getProperty(MessageConst.PROPERTY_CORRECTION_FLAG); + } + + + public static void setOriginMessageId(final Message msg, String OriginMessageId) { + putProperty(msg, MessageConst.PROPERTY_ORIGIN_MESSAGE_ID, OriginMessageId); + } + + + public static String getOriginMessageId(final Message msg) { + return msg.getProperty(MessageConst.PROPERTY_ORIGIN_MESSAGE_ID); + } + + + public static void setMQ2Flag(final Message msg, String flag) { + putProperty(msg, MessageConst.PROPERTY_MQ2_FLAG, flag); + } + + + public static String getMQ2Flag(final Message msg) { + return msg.getProperty(MessageConst.PROPERTY_MQ2_FLAG); + } + + + public static void setReconsumeTime(final Message msg, String reconsumeTimes) { + putProperty(msg, MessageConst.PROPERTY_RECONSUME_TIME, reconsumeTimes); + } + + + public static String getReconsumeTime(final Message msg) { + return msg.getProperty(MessageConst.PROPERTY_RECONSUME_TIME); + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/message/MessageConst.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/message/MessageConst.java new file mode 100644 index 000000000..e8f111811 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/message/MessageConst.java @@ -0,0 +1,78 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common.message; + +import java.util.HashSet; + + +public class MessageConst { + /** + * 消息关键词，多个Key用KEY_SEPARATOR隔开（查询消息使用） + */ + public static final String PROPERTY_KEYS = "KEYS"; + /** + * 消息标签，只支持设置一个Tag（服务端消息过滤使用） + */ + public static final String PROPERTY_TAGS = "TAGS"; + /** + * 是否等待服务器将消息存储完毕再返回（可能是等待刷盘完成或者等待同步复制到其他服务器） + */ + public static final String PROPERTY_WAIT_STORE_MSG_OK = "WAIT"; + /** + * 消息延时投递时间级别，0表示不延时，大于0表示特定延时级别（具体级别在服务器端定义） + */ + public static final String PROPERTY_DELAY_TIME_LEVEL = "DELAY"; + + /** + * 内部使用 + */ + public static final String PROPERTY_RETRY_TOPIC = "RETRY_TOPIC"; + public static final String PROPERTY_REAL_TOPIC = "REAL_TOPIC"; + public static final String PROPERTY_REAL_QUEUE_ID = "REAL_QID"; + public static final String PROPERTY_TRANSACTION_PREPARED = "TRAN_MSG"; + public static final String PROPERTY_PRODUCER_GROUP = "PGROUP"; + public static final String PROPERTY_MIN_OFFSET = "MIN_OFFSET"; + public static final String PROPERTY_MAX_OFFSET = "MAX_OFFSET"; + public static final String PROPERTY_BUYER_ID = "BUYER_ID"; + public static final String PROPERTY_ORIGIN_MESSAGE_ID = "ORIGIN_MESSAGE_ID"; + public static final String PROPERTY_TRANSFER_FLAG = "TRANSFER_FLAG"; + public static final String PROPERTY_CORRECTION_FLAG = "CORRECTION_FLAG"; + public static final String PROPERTY_MQ2_FLAG = "MQ2_FLAG"; + public static final String PROPERTY_RECONSUME_TIME = "RECONSUME_TIME"; + + public static final String KEY_SEPARATOR = " "; + + public static final HashSet systemKeySet = new HashSet(); + static { + systemKeySet.add(PROPERTY_KEYS); + systemKeySet.add(PROPERTY_TAGS); + systemKeySet.add(PROPERTY_WAIT_STORE_MSG_OK); + systemKeySet.add(PROPERTY_DELAY_TIME_LEVEL); + systemKeySet.add(PROPERTY_RETRY_TOPIC); + systemKeySet.add(PROPERTY_REAL_TOPIC); + systemKeySet.add(PROPERTY_REAL_QUEUE_ID); + systemKeySet.add(PROPERTY_TRANSACTION_PREPARED); + systemKeySet.add(PROPERTY_PRODUCER_GROUP); + systemKeySet.add(PROPERTY_MIN_OFFSET); + systemKeySet.add(PROPERTY_MAX_OFFSET); + systemKeySet.add(PROPERTY_BUYER_ID); + systemKeySet.add(PROPERTY_ORIGIN_MESSAGE_ID); + systemKeySet.add(PROPERTY_TRANSFER_FLAG); + systemKeySet.add(PROPERTY_CORRECTION_FLAG); + systemKeySet.add(PROPERTY_MQ2_FLAG); + systemKeySet.add(PROPERTY_RECONSUME_TIME); + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/message/MessageDecoder.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/message/MessageDecoder.java new file mode 100644 index 000000000..e35b16a02 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/message/MessageDecoder.java @@ -0,0 +1,284 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common.message; + +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.net.UnknownHostException; +import java.nio.BufferUnderflowException; +import java.nio.ByteBuffer; +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.sysflag.MessageSysFlag; + + +/** + * 消息解码 + * + * @author shijia.wxr + */ +public class MessageDecoder { + /** + * 消息ID定长 + */ + public final static int MSG_ID_LENGTH = 8 + 8; + + /** + * 存储记录各个字段位置 + */ + public final static int MessageMagicCodePostion = 4; + public final static int MessageFlagPostion = 16; + public final static int MessagePhysicOffsetPostion = 28; + public final static int MessageStoreTimestampPostion = 56; + + + public static String createMessageId(final ByteBuffer input, final ByteBuffer addr, final long offset) { + input.flip(); + input.limit(MessageDecoder.MSG_ID_LENGTH); + + // 消息存储主机地址 IP PORT 8 + input.put(addr); + // 消息对应的物理分区 OFFSET 8 + input.putLong(offset); + + return UtilAll.bytes2string(input.array()); + } + + + public static MessageId decodeMessageId(final String msgId) throws UnknownHostException { + SocketAddress address; + long offset; + + // 地址 + byte[] ip = UtilAll.string2bytes(msgId.substring(0, 8)); + byte[] port = UtilAll.string2bytes(msgId.substring(8, 16)); + ByteBuffer bb = ByteBuffer.wrap(port); + int portInt = bb.getInt(0); + address = new InetSocketAddress(InetAddress.getByAddress(ip), portInt); + + // offset + byte[] data = UtilAll.string2bytes(msgId.substring(16, 32)); + bb = ByteBuffer.wrap(data); + offset = bb.getLong(0); + + return new MessageId(address, offset); + } + + + public static MessageExt decode(java.nio.ByteBuffer byteBuffer) { + return decode(byteBuffer, true, true); + } + + + /** + * 客户端使用，SLAVE也会使用 + */ + public static MessageExt decode(java.nio.ByteBuffer byteBuffer, final boolean readBody) { + return decode(byteBuffer, readBody, true); + } + + + public static MessageExt decode(java.nio.ByteBuffer byteBuffer, final boolean readBody, + final boolean deCompressBody) { + try { + MessageExt msgExt = new MessageExt(); + + // 1 TOTALSIZE + int storeSize = byteBuffer.getInt(); + msgExt.setStoreSize(storeSize); + + // 2 MAGICCODE + byteBuffer.getInt(); + + // 3 BODYCRC + int bodyCRC = byteBuffer.getInt(); + msgExt.setBodyCRC(bodyCRC); + + // 4 QUEUEID + int queueId = byteBuffer.getInt(); + msgExt.setQueueId(queueId); + + // 5 FLAG + int flag = byteBuffer.getInt(); + msgExt.setFlag(flag); + + // 6 QUEUEOFFSET + long queueOffset = byteBuffer.getLong(); + msgExt.setQueueOffset(queueOffset); + + // 7 PHYSICALOFFSET + long physicOffset = byteBuffer.getLong(); + msgExt.setCommitLogOffset(physicOffset); + + // 8 SYSFLAG + int sysFlag = byteBuffer.getInt(); + msgExt.setSysFlag(sysFlag); + + // 9 BORNTIMESTAMP + long bornTimeStamp = byteBuffer.getLong(); + msgExt.setBornTimestamp(bornTimeStamp); + + // 10 BORNHOST + byte[] bornHost = new byte[4]; + byteBuffer.get(bornHost, 0, 4); + int port = byteBuffer.getInt(); + msgExt.setBornHost(new InetSocketAddress(InetAddress.getByAddress(bornHost), port)); + + // 11 STORETIMESTAMP + long storeTimestamp = byteBuffer.getLong(); + msgExt.setStoreTimestamp(storeTimestamp); + + // 12 STOREHOST + byte[] storeHost = new byte[4]; + byteBuffer.get(storeHost, 0, 4); + port = byteBuffer.getInt(); + msgExt.setStoreHost(new InetSocketAddress(InetAddress.getByAddress(storeHost), port)); + + // 13 RECONSUMETIMES + int reconsumeTimes = byteBuffer.getInt(); + msgExt.setReconsumeTimes(reconsumeTimes); + + // 14 Prepared Transaction Offset + long preparedTransactionOffset = byteBuffer.getLong(); + msgExt.setPreparedTransactionOffset(preparedTransactionOffset); + + // 15 BODY + int bodyLen = byteBuffer.getInt(); + if (bodyLen > 0) { + if (readBody) { + byte[] body = new byte[bodyLen]; + byteBuffer.get(body); + + // uncompress body + if (deCompressBody + && (sysFlag & MessageSysFlag.CompressedFlag) == MessageSysFlag.CompressedFlag) { + body = UtilAll.uncompress(body); + } + + msgExt.setBody(body); + } + else { + byteBuffer.position(byteBuffer.position() + bodyLen); + } + } + + // 16 TOPIC + byte topicLen = byteBuffer.get(); + byte[] topic = new byte[(int) topicLen]; + byteBuffer.get(topic); + msgExt.setTopic(new String(topic)); + + // 17 properties + short propertiesLength = byteBuffer.getShort(); + if (propertiesLength > 0) { + byte[] properties = new byte[propertiesLength]; + byteBuffer.get(properties); + String propertiesString = new String(properties, Charset.forName("UTF-8")); + Map map = string2messageProperties(propertiesString); + msgExt.setProperties(map); + } + + // 消息ID + ByteBuffer byteBufferMsgId = ByteBuffer.allocate(MSG_ID_LENGTH); + String msgId = + createMessageId(byteBufferMsgId, msgExt.getStoreHostBytes(), msgExt.getCommitLogOffset()); + msgExt.setMsgId(msgId); + + return msgExt; + } + catch (UnknownHostException e) { + byteBuffer.position(byteBuffer.limit()); + } + catch (BufferUnderflowException e) { + byteBuffer.position(byteBuffer.limit()); + } + catch (Exception e) { + byteBuffer.position(byteBuffer.limit()); + } + + return null; + } + + + public static List decodes(java.nio.ByteBuffer byteBuffer) { + return decodes(byteBuffer, true); + } + + + /** + * 客户端使用 + */ + public static List decodes(java.nio.ByteBuffer byteBuffer, final boolean readBody) { + List msgExts = new ArrayList(); + while (byteBuffer.hasRemaining()) { + MessageExt msgExt = decode(byteBuffer, readBody); + if (null != msgExt) { + msgExts.add(msgExt); + } + else { + break; + } + } + return msgExts; + } + + /** + * 序列化消息属性 + */ + public static final char NAME_VALUE_SEPARATOR = 1; + public static final char PROPERTY_SEPARATOR = 2; + + + public static String messageProperties2String(Map properties) { + StringBuilder sb = new StringBuilder(); + if (properties != null) { + for (final Map.Entry entry : properties.entrySet()) { + final String name = entry.getKey(); + final String value = entry.getValue(); + + sb.append(name); + sb.append(NAME_VALUE_SEPARATOR); + sb.append(value); + sb.append(PROPERTY_SEPARATOR); + } + } + return sb.toString(); + } + + + public static Map string2messageProperties(final String properties) { + Map map = new HashMap(); + if (properties != null) { + String[] items = properties.split(String.valueOf(PROPERTY_SEPARATOR)); + if (items != null) { + for (String i : items) { + String[] nv = i.split(String.valueOf(NAME_VALUE_SEPARATOR)); + if (nv != null && 2 == nv.length) { + map.put(nv[0], nv[1]); + } + } + } + } + + return map; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/message/MessageExt.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/message/MessageExt.java new file mode 100644 index 000000000..8185a07fa --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/message/MessageExt.java @@ -0,0 +1,275 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common.message; + +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.nio.ByteBuffer; + +import com.alibaba.rocketmq.common.TopicFilterType; +import com.alibaba.rocketmq.common.sysflag.MessageSysFlag; + + +/** + * 消息扩展属性，在服务器上产生此对象 + * + * @author shijia.wxr + * @since 2013-7-18 + */ +public class MessageExt extends Message { + private static final long serialVersionUID = 5720810158625748049L; + + // 队列ID + private int queueId; + // 存储记录大小 + private int storeSize; + // 队列偏移量 + private long queueOffset; + // 消息标志位 + private int sysFlag; + // 消息在客户端创建时间戳 + private long bornTimestamp; + // 消息来自哪里 + private SocketAddress bornHost; + // 消息在服务器存储时间戳 + private long storeTimestamp; + // 消息存储在哪个服务器 + private SocketAddress storeHost; + // 消息ID + private String msgId; + // 消息对应的Commit Log Offset + private long commitLogOffset; + // 消息体CRC + private int bodyCRC; + // 当前消息被某个订阅组重新消费了几次（订阅组之间独立计数） + private int reconsumeTimes; + + private long preparedTransactionOffset; + + + public MessageExt() { + } + + + public MessageExt(int queueId, long bornTimestamp, SocketAddress bornHost, long storeTimestamp, + SocketAddress storeHost, String msgId) { + this.queueId = queueId; + this.bornTimestamp = bornTimestamp; + this.bornHost = bornHost; + this.storeTimestamp = storeTimestamp; + this.storeHost = storeHost; + this.msgId = msgId; + } + + + /** + * SocketAddress ----> ByteBuffer 转化成8个字节 + */ + public static ByteBuffer SocketAddress2ByteBuffer(SocketAddress socketAddress) { + ByteBuffer byteBuffer = ByteBuffer.allocate(8); + InetSocketAddress inetSocketAddress = (InetSocketAddress) socketAddress; + byteBuffer.put(inetSocketAddress.getAddress().getAddress()); + byteBuffer.putInt(inetSocketAddress.getPort()); + byteBuffer.flip(); + return byteBuffer; + } + + + /** + * 获取bornHost字节形式，8个字节 HOST + PORT + */ + public ByteBuffer getBornHostBytes() { + return SocketAddress2ByteBuffer(this.bornHost); + } + + + /** + * 获取storehost字节形式，8个字节 HOST + PORT + */ + public ByteBuffer getStoreHostBytes() { + return SocketAddress2ByteBuffer(this.storeHost); + } + + + public int getQueueId() { + return queueId; + } + + + public void setQueueId(int queueId) { + this.queueId = queueId; + } + + + public long getBornTimestamp() { + return bornTimestamp; + } + + + public void setBornTimestamp(long bornTimestamp) { + this.bornTimestamp = bornTimestamp; + } + + + public SocketAddress getBornHost() { + return bornHost; + } + + + public String getBornHostString() { + if (this.bornHost != null) { + InetSocketAddress inetSocketAddress = (InetSocketAddress) this.bornHost; + return inetSocketAddress.getAddress().getHostAddress(); + } + + return null; + } + + + public String getBornHostNameString() { + if (this.bornHost != null) { + InetSocketAddress inetSocketAddress = (InetSocketAddress) this.bornHost; + return inetSocketAddress.getAddress().getHostName(); + } + + return null; + } + + + public void setBornHost(SocketAddress bornHost) { + this.bornHost = bornHost; + } + + + public long getStoreTimestamp() { + return storeTimestamp; + } + + + public void setStoreTimestamp(long storeTimestamp) { + this.storeTimestamp = storeTimestamp; + } + + + public SocketAddress getStoreHost() { + return storeHost; + } + + + public void setStoreHost(SocketAddress storeHost) { + this.storeHost = storeHost; + } + + + public String getMsgId() { + return msgId; + } + + + public void setMsgId(String msgId) { + this.msgId = msgId; + } + + + public int getSysFlag() { + return sysFlag; + } + + + public void setSysFlag(int sysFlag) { + this.sysFlag = sysFlag; + } + + + public int getBodyCRC() { + return bodyCRC; + } + + + public void setBodyCRC(int bodyCRC) { + this.bodyCRC = bodyCRC; + } + + + public long getQueueOffset() { + return queueOffset; + } + + + public void setQueueOffset(long queueOffset) { + this.queueOffset = queueOffset; + } + + + public long getCommitLogOffset() { + return commitLogOffset; + } + + + public void setCommitLogOffset(long physicOffset) { + this.commitLogOffset = physicOffset; + } + + + public int getStoreSize() { + return storeSize; + } + + + public void setStoreSize(int storeSize) { + this.storeSize = storeSize; + } + + + public static TopicFilterType parseTopicFilterType(final int sysFlag) { + if ((sysFlag & MessageSysFlag.MultiTagsFlag) == MessageSysFlag.MultiTagsFlag) { + return TopicFilterType.MULTI_TAG; + } + + return TopicFilterType.SINGLE_TAG; + } + + + public int getReconsumeTimes() { + return reconsumeTimes; + } + + + public void setReconsumeTimes(int reconsumeTimes) { + this.reconsumeTimes = reconsumeTimes; + } + + + public long getPreparedTransactionOffset() { + return preparedTransactionOffset; + } + + + public void setPreparedTransactionOffset(long preparedTransactionOffset) { + this.preparedTransactionOffset = preparedTransactionOffset; + } + + + @Override + public String toString() { + return "MessageExt [queueId=" + queueId + ", storeSize=" + storeSize + ", queueOffset=" + queueOffset + + ", sysFlag=" + sysFlag + ", bornTimestamp=" + bornTimestamp + ", bornHost=" + bornHost + + ", storeTimestamp=" + storeTimestamp + ", storeHost=" + storeHost + ", msgId=" + msgId + + ", commitLogOffset=" + commitLogOffset + ", bodyCRC=" + bodyCRC + ", reconsumeTimes=" + + reconsumeTimes + ", preparedTransactionOffset=" + preparedTransactionOffset + + ", toString()=" + super.toString() + "]"; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/message/MessageId.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/message/MessageId.java new file mode 100644 index 000000000..00ff09a01 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/message/MessageId.java @@ -0,0 +1,53 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common.message; + +import java.net.SocketAddress; + + +/** + * @author shijia.wxr + */ +public class MessageId { + private SocketAddress address; + private long offset; + + + public MessageId(SocketAddress address, long offset) { + this.address = address; + this.offset = offset; + } + + + public SocketAddress getAddress() { + return address; + } + + + public void setAddress(SocketAddress address) { + this.address = address; + } + + + public long getOffset() { + return offset; + } + + + public void setOffset(long offset) { + this.offset = offset; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/message/MessageQueue.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/message/MessageQueue.java new file mode 100644 index 000000000..c999fb921 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/message/MessageQueue.java @@ -0,0 +1,137 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common.message; + +import java.io.Serializable; + + +/** + * 消息队列数据结构，对外提供 + * + * @author shijia.wxr + */ +public class MessageQueue implements Comparable, Serializable { + private static final long serialVersionUID = 6191200464116433425L; + private String topic; + private String brokerName; + private int queueId; + + + public MessageQueue() { + + } + + + public MessageQueue(String topic, String brokerName, int queueId) { + this.topic = topic; + this.brokerName = brokerName; + this.queueId = queueId; + } + + + public String getTopic() { + return topic; + } + + + public void setTopic(String topic) { + this.topic = topic; + } + + + public String getBrokerName() { + return brokerName; + } + + + public void setBrokerName(String brokerName) { + this.brokerName = brokerName; + } + + + public int getQueueId() { + return queueId; + } + + + public void setQueueId(int queueId) { + this.queueId = queueId; + } + + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((brokerName == null) ? 0 : brokerName.hashCode()); + result = prime * result + queueId; + result = prime * result + ((topic == null) ? 0 : topic.hashCode()); + return result; + } + + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + MessageQueue other = (MessageQueue) obj; + if (brokerName == null) { + if (other.brokerName != null) + return false; + } + else if (!brokerName.equals(other.brokerName)) + return false; + if (queueId != other.queueId) + return false; + if (topic == null) { + if (other.topic != null) + return false; + } + else if (!topic.equals(other.topic)) + return false; + return true; + } + + + @Override + public String toString() { + return "MessageQueue [topic=" + topic + ", brokerName=" + brokerName + ", queueId=" + queueId + "]"; + } + + + @Override + public int compareTo(MessageQueue o) { + { + int result = this.topic.compareTo(o.topic); + if (result != 0) { + return result; + } + } + + { + int result = this.brokerName.compareTo(o.brokerName); + if (result != 0) { + return result; + } + } + + return this.queueId - o.queueId; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/namesrv/NamesrvConfig.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/namesrv/NamesrvConfig.java new file mode 100644 index 000000000..42725843f --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/namesrv/NamesrvConfig.java @@ -0,0 +1,43 @@ +/** + * $Id: NamesrvConfig.java 1839 2013-05-16 02:12:02Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.namesrv; + +import java.io.File; + +import com.alibaba.rocketmq.common.MixAll; + + +/** + * Name server 的配置类 + * + * @author shijia.wxr + * @author lansheng.zj@taobao.com + */ +public class NamesrvConfig { + private String rocketmqHome = System.getProperty(MixAll.ROCKETMQ_HOME_PROPERTY, + System.getenv(MixAll.ROCKETMQ_HOME_ENV)); + // 通用的KV配置持久化地址 + private String kvConfigPath = System.getProperty("user.home") + File.separator + "namesrv" + + File.separator + "kvConfig.json"; + + + public String getRocketmqHome() { + return rocketmqHome; + } + + + public void setRocketmqHome(String rocketmqHome) { + this.rocketmqHome = rocketmqHome; + } + + + public String getKvConfigPath() { + return kvConfigPath; + } + + + public void setKvConfigPath(String kvConfigPath) { + this.kvConfigPath = kvConfigPath; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/namesrv/NamesrvUtil.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/namesrv/NamesrvUtil.java new file mode 100644 index 000000000..84f696c9d --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/namesrv/NamesrvUtil.java @@ -0,0 +1,10 @@ +package com.alibaba.rocketmq.common.namesrv; + +/** + * @author shijia.wxr + * @since 2013-7-5 + */ +public class NamesrvUtil { + public static final String NAMESPACE_ORDER_TOPIC_CONFIG = "ORDER_TOPIC_CONFIG"; + public static final String NAMESPACE_PROJECT_CONFIG = "PROJECT_CONFIG"; +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/namesrv/RegisterBrokerResult.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/namesrv/RegisterBrokerResult.java new file mode 100644 index 000000000..977519873 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/namesrv/RegisterBrokerResult.java @@ -0,0 +1,44 @@ +package com.alibaba.rocketmq.common.namesrv; + +import com.alibaba.rocketmq.common.protocol.body.KVTable; + + +/** + * @author shijia.wxr + * @since 2013-7-8 + */ +public class RegisterBrokerResult { + private String haServerAddr; + private String masterAddr; + private KVTable kvTable; + + + public String getHaServerAddr() { + return haServerAddr; + } + + + public void setHaServerAddr(String haServerAddr) { + this.haServerAddr = haServerAddr; + } + + + public String getMasterAddr() { + return masterAddr; + } + + + public void setMasterAddr(String masterAddr) { + this.masterAddr = masterAddr; + } + + + public KVTable getKvTable() { + return kvTable; + } + + + public void setKvTable(KVTable kvTable) { + this.kvTable = kvTable; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/namesrv/TopAddressing.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/namesrv/TopAddressing.java new file mode 100644 index 000000000..28dbb93f1 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/namesrv/TopAddressing.java @@ -0,0 +1,90 @@ +/** + * $Id: TopAddressing.java 1831 2013-05-16 01:39:51Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.namesrv; + +import java.io.IOException; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.common.help.FAQUrl; +import com.alibaba.rocketmq.common.utils.HttpTinyClient; +import com.alibaba.rocketmq.common.utils.HttpTinyClient.HttpResult; + + +/** + * 寻址服务 + * + * @author shijia.wxr + * @author manhong.yqd + */ +public class TopAddressing { + private static final Logger log = LoggerFactory.getLogger(LoggerName.CommonLoggerName); + private String nsAddr; + private String wsAddr; + + + public TopAddressing(final String wsAddr) { + this.wsAddr = wsAddr; + } + + + private static String clearNewLine(final String str) { + String newString = str.trim(); + int index = newString.indexOf("\r"); + if (index != -1) { + return newString.substring(0, index); + } + + index = newString.indexOf("\n"); + if (index != -1) { + return newString.substring(0, index); + } + + return newString; + } + + + public final String fetchNSAddr() { + try { + HttpResult result = HttpTinyClient.httpGet(this.wsAddr, null, null, "UTF-8", 3000); + if (200 == result.code) { + String responseStr = result.content; + if (responseStr != null) { + return clearNewLine(responseStr); + } + else { + log.error("fetch nameserver address is null"); + } + } + else { + log.error("fetch nameserver address failed. statusCode={}", result.code); + } + } + catch (IOException e) { + log.error("fetchZKAddr exception", e); + } + + String errorMsg = + "connect to " + wsAddr + " failed, maybe the domain name " + MixAll.WS_DOMAIN_NAME + + " not bind in /etc/hosts"; + errorMsg += FAQUrl.suggestTodo(FAQUrl.NAME_SERVER_ADDR_NOT_EXIST_URL); + + log.warn(errorMsg); + System.out.println(errorMsg); + return null; + } + + + public String getNsAddr() { + return nsAddr; + } + + + public void setNsAddr(String nsAddr) { + this.nsAddr = nsAddr; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/MQProtosHelper.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/MQProtosHelper.java new file mode 100644 index 000000000..0aa728e7b --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/MQProtosHelper.java @@ -0,0 +1,49 @@ +package com.alibaba.rocketmq.common.protocol; + +import com.alibaba.rocketmq.common.protocol.header.namesrv.RegisterBrokerRequestHeader; +import com.alibaba.rocketmq.remoting.common.RemotingHelper; +import com.alibaba.rocketmq.remoting.exception.RemotingConnectException; +import com.alibaba.rocketmq.remoting.exception.RemotingSendRequestException; +import com.alibaba.rocketmq.remoting.exception.RemotingTimeoutException; +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; + + +/** + * 协议辅助类 + * + * @author shijia.wxr + */ +public class MQProtosHelper { + /** + * 将Broker地址注册到Name Server + */ + public static boolean registerBrokerToNameServer(final String nsaddr, final String brokerAddr, + final long timeoutMillis) { + RegisterBrokerRequestHeader requestHeader = new RegisterBrokerRequestHeader(); + requestHeader.setBrokerAddr(brokerAddr); + + RemotingCommand request = + RemotingCommand.createRequestCommand(RequestCode.REGISTER_BROKER, requestHeader); + + try { + RemotingCommand response = RemotingHelper.invokeSync(nsaddr, request, timeoutMillis); + if (response != null) { + return ResponseCode.SUCCESS == response.getCode(); + } + } + catch (RemotingConnectException e) { + e.printStackTrace(); + } + catch (RemotingSendRequestException e) { + e.printStackTrace(); + } + catch (RemotingTimeoutException e) { + e.printStackTrace(); + } + catch (InterruptedException e) { + e.printStackTrace(); + } + + return false; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/RequestCode.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/RequestCode.java new file mode 100644 index 000000000..f0314e307 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/RequestCode.java @@ -0,0 +1,170 @@ +package com.alibaba.rocketmq.common.protocol; + +public class RequestCode { + // Broker 发送消息 + public static final int SEND_MESSAGE = 10; + // Broker 订阅消息 + public static final int PULL_MESSAGE = 11; + // Broker 查询消息 + public static final int QUERY_MESSAGE = 12; + // Broker 查询Broker Offset + public static final int QUERY_BROKER_OFFSET = 13; + // Broker 查询Consumer Offset + public static final int QUERY_CONSUMER_OFFSET = 14; + // Broker 更新Consumer Offset + public static final int UPDATE_CONSUMER_OFFSET = 15; + // Broker 更新或者增加一个Topic + public static final int UPDATE_AND_CREATE_TOPIC = 17; + // Broker 获取所有Topic的配置（Slave和Namesrv都会向Master请求此配置） + public static final int GET_ALL_TOPIC_CONFIG = 21; + // Broker 获取所有Topic配置（Slave和Namesrv都会向Master请求此配置） + public static final int GET_TOPIC_CONFIG_LIST = 22; + // Broker 获取所有Topic名称列表 + public static final int GET_TOPIC_NAME_LIST = 23; + // Broker 更新Broker上的配置 + public static final int UPDATE_BROKER_CONFIG = 25; + // Broker 获取Broker上的配置 + public static final int GET_BROKER_CONFIG = 26; + // Broker 触发Broker删除文件 + public static final int TRIGGER_DELETE_FILES = 27; + // Broker 获取Broker运行时信息 + public static final int GET_BROKER_RUNTIME_INFO = 28; + // Broker 根据时间查询队列的Offset + public static final int SEARCH_OFFSET_BY_TIMESTAMP = 29; + // Broker 查询队列最大Offset + public static final int GET_MAX_OFFSET = 30; + // Broker 查询队列最小Offset + public static final int GET_MIN_OFFSET = 31; + // Broker 查询队列最早消息对应时间 + public static final int GET_EARLIEST_MSG_STORETIME = 32; + // Broker 根据消息ID来查询消息 + public static final int VIEW_MESSAGE_BY_ID = 33; + // Broker Client向Client发送心跳，并注册自身 + public static final int HEART_BEAT = 34; + // Broker Client注销 + public static final int UNREGISTER_CLIENT = 35; + // Broker Consumer将处理不了的消息发回服务器 + public static final int CONSUMER_SEND_MSG_BACK = 36; + // Broker Commit或者Rollback事务 + public static final int END_TRANSACTION = 37; + // Broker 获取ConsumerId列表通过GroupName + public static final int GET_CONSUMER_LIST_BY_GROUP = 38; + // Broker 主动向Producer回查事务状态 + public static final int CHECK_TRANSACTION_STATE = 39; + // Broker Broker通知Consumer列表变化 + public static final int NOTIFY_CONSUMER_IDS_CHANGED = 40; + // Broker Consumer向Master锁定队列 + public static final int LOCK_BATCH_MQ = 41; + // Broker Consumer向Master解锁队列 + public static final int UNLOCK_BATCH_MQ = 42; + // Broker 获取所有Consumer Offset + public static final int GET_ALL_CONSUMER_OFFSET = 43; + // Broker 获取所有定时进度 + public static final int GET_ALL_DELAY_OFFSET = 45; + // Namesrv 向Namesrv追加KV配置 + public static final int PUT_KV_CONFIG = 100; + // Namesrv 从Namesrv获取KV配置 + public static final int GET_KV_CONFIG = 101; + // Namesrv 从Namesrv获取KV配置 + public static final int DELETE_KV_CONFIG = 102; + // Namesrv 注册一个Broker，数据都是持久化的，如果存在则覆盖配置 + public static final int REGISTER_BROKER = 103; + // Namesrv 卸载一个Broker，数据都是持久化的 + public static final int UNREGISTER_BROKER = 104; + // Namesrv 根据Topic获取Broker Name、队列数(包含读队列与写队列) + public static final int GET_ROUTEINTO_BY_TOPIC = 105; + // Namesrv 获取注册到Name Server的所有Broker集群信息 + public static final int GET_BROKER_CLUSTER_INFO = 106; + public static final int UPDATE_AND_CREATE_SUBSCRIPTIONGROUP = 200; + public static final int GET_ALL_SUBSCRIPTIONGROUP_CONFIG = 201; + public static final int GET_TOPIC_STATS_INFO = 202; + public static final int GET_CONSUMER_CONNECTION_LIST = 203; + public static final int GET_PRODUCER_CONNECTION_LIST = 204; + public static final int WIPE_WRITE_PERM_OF_BROKER = 205; + + // 从Name Server获取完整Topic列表 + public static final int GET_ALL_TOPIC_LIST_FROM_NAMESERVER = 206; + // 从Broker删除订阅组 + public static final int DELETE_SUBSCRIPTIONGROUP = 207; + // 从Broker获取消费状态（进度） + public static final int GET_CONSUME_STATS = 208; + // Suspend Consumer消费过程 + public static final int SUSPEND_CONSUMER = 209; + // Resume Consumer消费过程 + public static final int RESUME_CONSUMER = 210; + // 重置Consumer Offset + public static final int RESET_CONSUMER_OFFSET_IN_CONSUMER = 211; + // 重置Consumer Offset + public static final int RESET_CONSUMER_OFFSET_IN_BROKER = 212; + // 调整Consumer线程池数量 + public static final int ADJUST_CONSUMER_THREAD_POOL = 213; + // 查询消息被哪些消费组消费 + public static final int WHO_CONSUME_THE_MESSAGE = 214; + + // 从Broker删除Topic配置 + public static final int DELETE_TOPIC_IN_BROKER = 215; + // 从Namesrv删除Topic配置 + public static final int DELETE_TOPIC_IN_NAMESRV = 216; + // Namesrv 通过 project 获取所有的 server ip 信息 + public static final int GET_KV_CONFIG_BY_VALUE = 217; + // Namesrv 删除指定 project group 下的所有 server ip 信息 + public static final int DELETE_KV_CONFIG_BY_VALUE = 218; + // 通过NameSpace获取所有的KV List + public static final int GET_KVLIST_BY_NAMESPACE = 219; + + // offset 重置 + public static final int RESET_CONSUMER_CLIENT_OFFSET = 220; + // 客户端订阅消息 + public static final int GET_CONSUMER_STATUS_FROM_CLIENT = 221; + // 通知 broker 调用 offset 重置处理 + public static final int INVOKE_BROKER_TO_RESET_OFFSET = 222; + // 通知 broker 调用客户端订阅消息处理 + public static final int INVOKE_BROKER_TO_GET_CONSUMER_STATUS = 223; + + // Broker 查询topic被谁消费 + // 2014-03-21 Add By shijia + public static final int QUERY_TOPIC_CONSUME_BY_WHO = 300; + + // 获取指定集群下的所有 topic + // 2014-03-26 + public static final int GET_TOPICS_BY_CLUSTER = 224; + + // 向Broker注册Filter Server + // 2014-04-06 Add By shijia + public static final int REGISTER_FILTER_SERVER = 301; + // 向Filter Server注册Class + // 2014-04-06 Add By shijia + public static final int REGISTER_MESSAGE_FILTER_CLASS = 302; + // 根据 topic 和 group 获取消息的时间跨度 + public static final int QUERY_CONSUME_TIME_SPAN = 303; + // 获取所有系统内置 Topic 列表 + public static final int GET_SYSTEM_TOPIC_LIST_FROM_NS = 304; + public static final int GET_SYSTEM_TOPIC_LIST_FROM_BROKER = 305; + + // 清理失效队列 + public static final int CLEAN_EXPIRED_CONSUMEQUEUE = 306; + + // 通过Broker查询Consumer内存数据 + // 2014-07-19 Add By shijia + public static final int GET_CONSUMER_RUNNING_INFO = 307; + + // 查找被修正 offset (转发组件） + public static final int QUERY_CORRECTION_OFFSET = 308; + + // 通过Broker直接向某个Consumer发送一条消息，并立刻消费，返回结果给broker，再返回给调用方 + // 2014-08-11 Add By shijia + public static final int CONSUME_MESSAGE_DIRECTLY = 309; + + // Broker 发送消息，优化网络数据包 + public static final int SEND_MESSAGE_V2 = 310; + + // 单元化相关 topic + public static final int GET_UNIT_TOPIC_LIST = 311; + // 获取含有单元化订阅组的 Topic 列表 + public static final int GET_HAS_UNIT_SUB_TOPIC_LIST = 312; + // 获取含有单元化订阅组的非单元化 Topic 列表 + public static final int GET_HAS_UNIT_SUB_UNUNIT_TOPIC_LIST = 313; + // 克隆某一个组的消费进度到新的组 + public static final int CLONE_GROUP_OFFSET = 314; + +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/ResponseCode.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/ResponseCode.java new file mode 100644 index 000000000..4288601e8 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/ResponseCode.java @@ -0,0 +1,60 @@ +package com.alibaba.rocketmq.common.protocol; + +import com.alibaba.rocketmq.remoting.protocol.RemotingSysResponseCode; + + +public class ResponseCode extends RemotingSysResponseCode { + // Broker 刷盘超时 + public static final int FLUSH_DISK_TIMEOUT = 10; + // Broker 同步双写，Slave不可用 + public static final int SLAVE_NOT_AVAILABLE = 11; + // Broker 同步双写，等待Slave应答超时 + public static final int FLUSH_SLAVE_TIMEOUT = 12; + // Broker 消息非法 + public static final int MESSAGE_ILLEGAL = 13; + // Broker, Namesrv 服务不可用，可能是正在关闭或者权限问题 + public static final int SERVICE_NOT_AVAILABLE = 14; + // Broker, Namesrv 版本号不支持 + public static final int VERSION_NOT_SUPPORTED = 15; + // Broker, Namesrv 无权限执行此操作，可能是发、收、或者其他操作 + public static final int NO_PERMISSION = 16; + // Broker, Topic不存在 + public static final int TOPIC_NOT_EXIST = 17; + // Broker, Topic已经存在，创建Topic + public static final int TOPIC_EXIST_ALREADY = 18; + // Broker 拉消息未找到（请求的Offset等于最大Offset，最大Offset无对应消息） + public static final int PULL_NOT_FOUND = 19; + // Broker 可能被过滤，或者误通知等 + public static final int PULL_RETRY_IMMEDIATELY = 20; + // Broker 拉消息请求的Offset不合法，太小或太大 + public static final int PULL_OFFSET_MOVED = 21; + // Broker 查询消息未找到 + public static final int QUERY_NOT_FOUND = 22; + // Broker 订阅关系解析失败 + public static final int SUBSCRIPTION_PARSE_FAILED = 23; + // Broker 订阅关系不存在 + public static final int SUBSCRIPTION_NOT_EXIST = 24; + // Broker 订阅关系不是最新的 + public static final int SUBSCRIPTION_NOT_LATEST = 25; + // Broker 订阅组不存在 + public static final int SUBSCRIPTION_GROUP_NOT_EXIST = 26; + // Producer 事务应该被提交 + public static final int TRANSACTION_SHOULD_COMMIT = 200; + // Producer 事务应该被回滚 + public static final int TRANSACTION_SHOULD_ROLLBACK = 201; + // Producer 事务状态未知 + public static final int TRANSACTION_STATE_UNKNOW = 202; + // Producer ProducerGroup错误 + public static final int TRANSACTION_STATE_GROUP_WRONG = 203; + // 单元化消息，需要设置 buyerId + public static final int NO_BUYER_ID = 204; + + // 单元化消息，非本单元消息 + public static final int NOT_IN_CURRENT_UNIT = 205; + + // Consumer不在线 + public static final int CONSUMER_NOT_ONLINE = 206; + + // Consumer消费消息超时 + public static final int CONSUME_MSG_TIMEOUT = 207; +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/CMResult.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/CMResult.java new file mode 100644 index 000000000..28d42b164 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/CMResult.java @@ -0,0 +1,10 @@ +package com.alibaba.rocketmq.common.protocol.body; + +public enum CMResult { + CR_SUCCESS, + CR_LATER, + CR_ROLLBACK, + CR_COMMIT, + CR_THROW_EXCEPTION, + CR_RETURN_NULL, +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ClusterInfo.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ClusterInfo.java new file mode 100644 index 000000000..f4f8821cf --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ClusterInfo.java @@ -0,0 +1,62 @@ +package com.alibaba.rocketmq.common.protocol.body; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Set; + +import com.alibaba.rocketmq.common.protocol.route.BrokerData; +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; + + +/** + * 协议中传输对象，内容为集群信息 + * + * @author shijia.wxr + * @since 2013-7-16 + */ +public class ClusterInfo extends RemotingSerializable { + private HashMap brokerAddrTable; + private HashMap> clusterAddrTable; + + + public HashMap getBrokerAddrTable() { + return brokerAddrTable; + } + + + public void setBrokerAddrTable(HashMap brokerAddrTable) { + this.brokerAddrTable = brokerAddrTable; + } + + + public HashMap> getClusterAddrTable() { + return clusterAddrTable; + } + + + public void setClusterAddrTable(HashMap> clusterAddrTable) { + this.clusterAddrTable = clusterAddrTable; + } + + + public String[] retrieveAllAddrByCluster(String cluster) { + List addrs = new ArrayList(); + if (clusterAddrTable.containsKey(cluster)) { + Set brokerNames = clusterAddrTable.get(cluster); + for (String brokerName : brokerNames) { + BrokerData brokerData = brokerAddrTable.get(brokerName); + if (null != brokerData) { + addrs.addAll(brokerData.getBrokerAddrs().values()); + } + } + } + + return addrs.toArray(new String[] {}); + } + + + public String[] retrieveAllClusterNames() { + return clusterAddrTable.keySet().toArray(new String[] {}); + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/Connection.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/Connection.java new file mode 100644 index 000000000..f2ca90c13 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/Connection.java @@ -0,0 +1,57 @@ +package com.alibaba.rocketmq.common.protocol.body; + +import com.alibaba.rocketmq.remoting.protocol.LanguageCode; + + +/** + * TODO + * + * @author shijia.wxr + * @since 13-8-5 + */ +public class Connection { + private String clientId; + private String clientAddr; + private LanguageCode language; + private int version; + + + public String getClientId() { + return clientId; + } + + + public void setClientId(String clientId) { + this.clientId = clientId; + } + + + public String getClientAddr() { + return clientAddr; + } + + + public void setClientAddr(String clientAddr) { + this.clientAddr = clientAddr; + } + + + public LanguageCode getLanguage() { + return language; + } + + + public void setLanguage(LanguageCode language) { + this.language = language; + } + + + public int getVersion() { + return version; + } + + + public void setVersion(int version) { + this.version = version; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ConsumeByWho.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ConsumeByWho.java new file mode 100644 index 000000000..c7fc20ffc --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ConsumeByWho.java @@ -0,0 +1,83 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common.protocol.body; + +import java.util.HashSet; + +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; + + +/** + * @author shijia.wxr + * @since 2013-8-10 + */ +public class ConsumeByWho extends RemotingSerializable { + private HashSet consumedGroup = new HashSet(); + private HashSet notConsumedGroup = new HashSet(); + private String topic; + private int queueId; + private long offset; + + + public HashSet getConsumedGroup() { + return consumedGroup; + } + + + public void setConsumedGroup(HashSet consumedGroup) { + this.consumedGroup = consumedGroup; + } + + + public HashSet getNotConsumedGroup() { + return notConsumedGroup; + } + + + public void setNotConsumedGroup(HashSet notConsumedGroup) { + this.notConsumedGroup = notConsumedGroup; + } + + + public String getTopic() { + return topic; + } + + + public void setTopic(String topic) { + this.topic = topic; + } + + + public int getQueueId() { + return queueId; + } + + + public void setQueueId(int queueId) { + this.queueId = queueId; + } + + + public long getOffset() { + return offset; + } + + + public void setOffset(long offset) { + this.offset = offset; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ConsumeMessageDirectlyResult.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ConsumeMessageDirectlyResult.java new file mode 100644 index 000000000..00b3159d6 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ConsumeMessageDirectlyResult.java @@ -0,0 +1,70 @@ +package com.alibaba.rocketmq.common.protocol.body; + +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; + + +public class ConsumeMessageDirectlyResult extends RemotingSerializable { + private boolean order = false; + private boolean autoCommit = true; + private CMResult consumeResult; + private String remark; + private long spentTimeMills; + + + public boolean isOrder() { + return order; + } + + + public void setOrder(boolean order) { + this.order = order; + } + + + public boolean isAutoCommit() { + return autoCommit; + } + + + public void setAutoCommit(boolean autoCommit) { + this.autoCommit = autoCommit; + } + + + public String getRemark() { + return remark; + } + + + public void setRemark(String remark) { + this.remark = remark; + } + + + public CMResult getConsumeResult() { + return consumeResult; + } + + + public void setConsumeResult(CMResult consumeResult) { + this.consumeResult = consumeResult; + } + + + public long getSpentTimeMills() { + return spentTimeMills; + } + + + public void setSpentTimeMills(long spentTimeMills) { + this.spentTimeMills = spentTimeMills; + } + + + @Override + public String toString() { + return "ConsumeMessageDirectlyResult [order=" + order + ", autoCommit=" + autoCommit + + ", consumeResult=" + consumeResult + ", remark=" + remark + ", spentTimeMills=" + + spentTimeMills + "]"; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ConsumeStatus.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ConsumeStatus.java new file mode 100644 index 000000000..551fc5756 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ConsumeStatus.java @@ -0,0 +1,74 @@ +package com.alibaba.rocketmq.common.protocol.body; + +/** + * 消费过程的统计数据 + */ +public class ConsumeStatus { + private double pullRT; + private double pullTPS; + private double consumeRT; + private double consumeOKTPS; + private double consumeFailedTPS; + // 最近一小时内消费失败的消息数 + private long consumeFailedMsgs; + + + public double getPullRT() { + return pullRT; + } + + + public void setPullRT(double pullRT) { + this.pullRT = pullRT; + } + + + public double getPullTPS() { + return pullTPS; + } + + + public void setPullTPS(double pullTPS) { + this.pullTPS = pullTPS; + } + + + public double getConsumeRT() { + return consumeRT; + } + + + public void setConsumeRT(double consumeRT) { + this.consumeRT = consumeRT; + } + + + public double getConsumeOKTPS() { + return consumeOKTPS; + } + + + public void setConsumeOKTPS(double consumeOKTPS) { + this.consumeOKTPS = consumeOKTPS; + } + + + public double getConsumeFailedTPS() { + return consumeFailedTPS; + } + + + public void setConsumeFailedTPS(double consumeFailedTPS) { + this.consumeFailedTPS = consumeFailedTPS; + } + + + public long getConsumeFailedMsgs() { + return consumeFailedMsgs; + } + + + public void setConsumeFailedMsgs(long consumeFailedMsgs) { + this.consumeFailedMsgs = consumeFailedMsgs; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ConsumerConnection.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ConsumerConnection.java new file mode 100644 index 000000000..104149c1b --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ConsumerConnection.java @@ -0,0 +1,88 @@ +package com.alibaba.rocketmq.common.protocol.body; + +import java.util.HashSet; +import java.util.concurrent.ConcurrentHashMap; + +import com.alibaba.rocketmq.common.consumer.ConsumeFromWhere; +import com.alibaba.rocketmq.common.protocol.heartbeat.ConsumeType; +import com.alibaba.rocketmq.common.protocol.heartbeat.MessageModel; +import com.alibaba.rocketmq.common.protocol.heartbeat.SubscriptionData; +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; + + +/** + * TODO + * + * @author shijia.wxr + * @since 13-8-5 + */ +public class ConsumerConnection extends RemotingSerializable { + private HashSet connectionSet = new HashSet(); + private ConcurrentHashMap subscriptionTable = + new ConcurrentHashMap(); + private ConsumeType consumeType; + private MessageModel messageModel; + private ConsumeFromWhere consumeFromWhere; + + + public int computeMinVersion() { + int minVersion = Integer.MAX_VALUE; + for (Connection c : this.connectionSet) { + if (c.getVersion() < minVersion) { + minVersion = c.getVersion(); + } + } + + return minVersion; + } + + + public HashSet getConnectionSet() { + return connectionSet; + } + + + public void setConnectionSet(HashSet connectionSet) { + this.connectionSet = connectionSet; + } + + + public ConcurrentHashMap getSubscriptionTable() { + return subscriptionTable; + } + + + public void setSubscriptionTable(ConcurrentHashMap subscriptionTable) { + this.subscriptionTable = subscriptionTable; + } + + + public ConsumeType getConsumeType() { + return consumeType; + } + + + public void setConsumeType(ConsumeType consumeType) { + this.consumeType = consumeType; + } + + + public MessageModel getMessageModel() { + return messageModel; + } + + + public void setMessageModel(MessageModel messageModel) { + this.messageModel = messageModel; + } + + + public ConsumeFromWhere getConsumeFromWhere() { + return consumeFromWhere; + } + + + public void setConsumeFromWhere(ConsumeFromWhere consumeFromWhere) { + this.consumeFromWhere = consumeFromWhere; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ConsumerOffsetSerializeWrapper.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ConsumerOffsetSerializeWrapper.java new file mode 100644 index 000000000..bf0177952 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ConsumerOffsetSerializeWrapper.java @@ -0,0 +1,27 @@ +package com.alibaba.rocketmq.common.protocol.body; + +import java.util.concurrent.ConcurrentHashMap; + +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; + + +/** + * Consumer消费进度，序列化包装 + * + * @author manhong.yqd + * @since 2013-8-19 + */ +public class ConsumerOffsetSerializeWrapper extends RemotingSerializable { + private ConcurrentHashMap> offsetTable = + new ConcurrentHashMap>(512); + + + public ConcurrentHashMap> getOffsetTable() { + return offsetTable; + } + + + public void setOffsetTable(ConcurrentHashMap> offsetTable) { + this.offsetTable = offsetTable; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ConsumerRunningInfo.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ConsumerRunningInfo.java new file mode 100644 index 000000000..53d29e170 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ConsumerRunningInfo.java @@ -0,0 +1,323 @@ +package com.alibaba.rocketmq.common.protocol.body; + +import java.util.Iterator; +import java.util.Map.Entry; +import java.util.Properties; +import java.util.TreeMap; +import java.util.TreeSet; + +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.common.protocol.heartbeat.ConsumeType; +import com.alibaba.rocketmq.common.protocol.heartbeat.SubscriptionData; +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; + + +/** + * Consumer内部数据结构 + */ +public class ConsumerRunningInfo extends RemotingSerializable { + public static final String PROP_NAMESERVER_ADDR = "PROP_NAMESERVER_ADDR"; + public static final String PROP_THREADPOOL_CORE_SIZE = "PROP_THREADPOOL_CORE_SIZE"; + public static final String PROP_CONSUME_ORDERLY = "PROP_CONSUMEORDERLY"; + public static final String PROP_CONSUME_TYPE = "PROP_CONSUME_TYPE"; + public static final String PROP_CLIENT_VERSION = "PROP_CLIENT_VERSION"; + public static final String PROP_CONSUMER_START_TIMESTAMP = "PROP_CONSUMER_START_TIMESTAMP"; + + // 各种配置及运行数据 + private Properties properties = new Properties(); + // 订阅关系 + private TreeSet subscriptionSet = new TreeSet(); + // 消费进度、Rebalance、内部消费队列的信息 + private TreeMap mqTable = new TreeMap(); + // RT、TPS统计 + private TreeMap statusTable = new TreeMap(); + // jstack的结果 + private String jstack; + + + public Properties getProperties() { + return properties; + } + + + public void setProperties(Properties properties) { + this.properties = properties; + } + + + public TreeMap getMqTable() { + return mqTable; + } + + + public void setMqTable(TreeMap mqTable) { + this.mqTable = mqTable; + } + + + public TreeMap getStatusTable() { + return statusTable; + } + + + public void setStatusTable(TreeMap statusTable) { + this.statusTable = statusTable; + } + + + public TreeSet getSubscriptionSet() { + return subscriptionSet; + } + + + public void setSubscriptionSet(TreeSet subscriptionSet) { + this.subscriptionSet = subscriptionSet; + } + + + public String formatString() { + StringBuilder sb = new StringBuilder(); + + // 1 + { + sb.append("#Consumer Properties#\n"); + Iterator> it = this.properties.entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + String item = + String.format("%-40s: %s\n", next.getKey().toString(), next.getValue().toString()); + sb.append(item); + } + } + + // 2 + { + sb.append("\n\n#Consumer Subscription#\n"); + + Iterator it = this.subscriptionSet.iterator(); + int i = 0; + while (it.hasNext()) { + SubscriptionData next = it.next(); + String item = String.format("%03d Topic: %-40s ClassFilter: %-8s SubExpression: %s\n", // + ++i,// + next.getTopic(),// + next.isClassFilterMode(),// + next.getSubString()); + + sb.append(item); + } + } + + // 3 + { + sb.append("\n\n#Consumer Offset#\n"); + sb.append(String.format("%-32s %-32s %-4s %-20s\n",// + "#Topic",// + "#Broker Name",// + "#QID",// + "#Consumer Offset"// + )); + + Iterator> it = this.mqTable.entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + String item = String.format("%-32s %-32s %-4d %-20d\n",// + next.getKey().getTopic(),// + next.getKey().getBrokerName(),// + next.getKey().getQueueId(),// + next.getValue().getCommitOffset()); + + sb.append(item); + } + } + + // 4 + { + sb.append("\n\n#Consumer MQ Detail#\n"); + sb.append(String.format("%-32s %-32s %-4s %-20s\n",// + "#Topic",// + "#Broker Name",// + "#QID",// + "#ProcessQueueInfo"// + )); + + Iterator> it = this.mqTable.entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + String item = String.format("%-32s %-32s %-4d %s\n",// + next.getKey().getTopic(),// + next.getKey().getBrokerName(),// + next.getKey().getQueueId(),// + next.getValue().toString()); + + sb.append(item); + } + } + + // 5 + { + sb.append("\n\n#Consumer RT&TPS#\n"); + sb.append(String.format("%-32s %14s %14s %14s %14s %18s %25s\n",// + "#Topic",// + "#Pull RT",// + "#Pull TPS",// + "#Consume RT",// + "#ConsumeOK TPS",// + "#ConsumeFailed TPS",// + "#ConsumeFailedMsgsInHour"// + )); + + Iterator> it = this.statusTable.entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + String item = String.format("%-32s %14.2f %14.2f %14.2f %14.2f %18.2f %25d\n",// + next.getKey(),// + next.getValue().getPullRT(),// + next.getValue().getPullTPS(),// + next.getValue().getConsumeRT(),// + next.getValue().getConsumeOKTPS(),// + next.getValue().getConsumeFailedTPS(),// + next.getValue().getConsumeFailedMsgs()// + ); + + sb.append(item); + } + } + + // 6 + if (this.jstack != null) { + sb.append("\n\n#Consumer jstack#\n"); + sb.append(this.jstack); + } + + return sb.toString(); + } + + + /** + * 分析订阅关系是否相同 + */ + public static boolean analyzeSubscription( + final TreeMap criTable) { + ConsumerRunningInfo prev = criTable.firstEntry().getValue(); + + boolean push = false; + { + String property = prev.getProperties().getProperty(ConsumerRunningInfo.PROP_CONSUME_TYPE); + push = ConsumeType.valueOf(property) == ConsumeType.CONSUME_PASSIVELY; + } + + boolean startForAWhile = false; + { + String property = + prev.getProperties().getProperty(ConsumerRunningInfo.PROP_CONSUMER_START_TIMESTAMP); + startForAWhile = (System.currentTimeMillis() - Long.parseLong(property)) > (1000 * 60 * 2); + } + + // 只检测PUSH + if (push && startForAWhile) { + // 分析订阅关系是否相同 + { + Iterator> it = criTable.entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + ConsumerRunningInfo current = next.getValue(); + boolean equals = current.getSubscriptionSet().equals(prev.getSubscriptionSet()); + // 发现订阅关系有误 + if (!equals) { + // Different subscription in the same group of consumer + return false; + } + + prev = next.getValue(); + } + + if (prev != null) { + // 无订阅关系 + if (prev.getSubscriptionSet().isEmpty()) { + // Subscription empty! + return false; + } + } + } + } + + return true; + } + + + public static boolean analyzeRebalance(final TreeMap criTable) { + return true; + } + + + public static String analyzeProcessQueue(final String clientId, ConsumerRunningInfo info) { + StringBuilder sb = new StringBuilder(); + boolean push = false; + { + String property = info.getProperties().getProperty(ConsumerRunningInfo.PROP_CONSUME_TYPE); + push = ConsumeType.valueOf(property) == ConsumeType.CONSUME_PASSIVELY; + } + + boolean orderMsg = false; + { + String property = info.getProperties().getProperty(ConsumerRunningInfo.PROP_CONSUME_ORDERLY); + orderMsg = Boolean.parseBoolean(property); + } + + if (push) { + Iterator> it = info.getMqTable().entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + MessageQueue mq = next.getKey(); + ProcessQueueInfo pq = next.getValue(); + + // 顺序消息 + if (orderMsg) { + // 没锁住 + if (!pq.isLocked()) { + sb.append(String.format("%s %s can't lock for a while, %dms\n", // + clientId,// + mq,// + System.currentTimeMillis() - pq.getLastLockTimestamp())); + } + // 锁住 + else { + // Rebalance已经丢弃此队列，但是没有正常释放Lock + if (pq.isDroped() && (pq.getTryUnlockTimes() > 0)) { + sb.append(String.format("%s %s unlock %d times, still failed\n",// + clientId,// + mq,// + pq.getTryUnlockTimes())); + } + } + + // 事务消息未提交 + } + // 乱序消息 + else { + long diff = System.currentTimeMillis() - pq.getLastConsumeTimestamp(); + // 在有消息的情况下，超过1分钟没再消费消息了 + if (diff > (1000 * 60) && pq.getCachedMsgCount() > 0) { + sb.append(String.format("%s %s can't consume for a while, maybe blocked, %dms\n",// + clientId,// + mq, // + diff)); + } + } + } + } + + return sb.toString(); + } + + + public String getJstack() { + return jstack; + } + + + public void setJstack(String jstack) { + this.jstack = jstack; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/GetConsumerStatusBody.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/GetConsumerStatusBody.java new file mode 100644 index 000000000..09f64b8c3 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/GetConsumerStatusBody.java @@ -0,0 +1,41 @@ +package com.alibaba.rocketmq.common.protocol.body; + +import java.util.HashMap; +import java.util.Map; + +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; + + +/** + * 查看客户端消费组的消费情况。 + * + * @author: manhong.yqd + * @since: 13-12-30 + */ +@Deprecated +public class GetConsumerStatusBody extends RemotingSerializable { + private Map messageQueueTable = new HashMap(); + private Map> consumerTable = + new HashMap>(); + + + public Map getMessageQueueTable() { + return messageQueueTable; + } + + + public void setMessageQueueTable(Map messageQueueTable) { + this.messageQueueTable = messageQueueTable; + } + + + public Map> getConsumerTable() { + return consumerTable; + } + + + public void setConsumerTable(Map> consumerTable) { + this.consumerTable = consumerTable; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/GroupList.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/GroupList.java new file mode 100644 index 000000000..d466beb9c --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/GroupList.java @@ -0,0 +1,39 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common.protocol.body; + +import java.util.HashSet; + +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; + + +/** + * @author shijia.wxr + * @since 2014-3-20 + */ +public class GroupList extends RemotingSerializable { + private HashSet groupList = new HashSet(); + + + public HashSet getGroupList() { + return groupList; + } + + + public void setGroupList(HashSet groupList) { + this.groupList = groupList; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/KVTable.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/KVTable.java new file mode 100644 index 000000000..c76ea974a --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/KVTable.java @@ -0,0 +1,39 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common.protocol.body; + +import java.util.HashMap; + +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; + + +/** + * @author shijia.wxr + * @since 2013-8-14 + */ +public class KVTable extends RemotingSerializable { + private HashMap table = new HashMap(); + + + public HashMap getTable() { + return table; + } + + + public void setTable(HashMap table) { + this.table = table; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/LockBatchRequestBody.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/LockBatchRequestBody.java new file mode 100644 index 000000000..4f9a9e76f --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/LockBatchRequestBody.java @@ -0,0 +1,48 @@ +package com.alibaba.rocketmq.common.protocol.body; + +import java.util.HashSet; +import java.util.Set; + +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; + + +/** + * @author shijia.wxr + * @since 2013-6-26 + */ +public class LockBatchRequestBody extends RemotingSerializable { + private String consumerGroup; + private String clientId; + private Set mqSet = new HashSet(); + + + public String getConsumerGroup() { + return consumerGroup; + } + + + public void setConsumerGroup(String consumerGroup) { + this.consumerGroup = consumerGroup; + } + + + public String getClientId() { + return clientId; + } + + + public void setClientId(String clientId) { + this.clientId = clientId; + } + + + public Set getMqSet() { + return mqSet; + } + + + public void setMqSet(Set mqSet) { + this.mqSet = mqSet; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/LockBatchResponseBody.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/LockBatchResponseBody.java new file mode 100644 index 000000000..f959ad11c --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/LockBatchResponseBody.java @@ -0,0 +1,28 @@ +package com.alibaba.rocketmq.common.protocol.body; + +import java.util.HashSet; +import java.util.Set; + +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; + + +/** + * @author shijia.wxr + * @since 2013-6-26 + */ +public class LockBatchResponseBody extends RemotingSerializable { + // Lock成功的队列集合 + private Set lockOKMQSet = new HashSet(); + + + public Set getLockOKMQSet() { + return lockOKMQSet; + } + + + public void setLockOKMQSet(Set lockOKMQSet) { + this.lockOKMQSet = lockOKMQSet; + } + +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ProcessQueueInfo.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ProcessQueueInfo.java new file mode 100644 index 000000000..77a552fa2 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ProcessQueueInfo.java @@ -0,0 +1,180 @@ +package com.alibaba.rocketmq.common.protocol.body; + +import com.alibaba.rocketmq.common.UtilAll; + + +public class ProcessQueueInfo { + /** + * 消费到哪里，提交的offset + */ + private long commitOffset; + + /** + * 缓存的消息Offset信息 + */ + private long cachedMsgMinOffset; + private long cachedMsgMaxOffset; + private int cachedMsgCount; + + /** + * 正在事务中的消息 + */ + private long transactionMsgMinOffset; + private long transactionMsgMaxOffset; + private int transactionMsgCount; + + /** + * 顺序消息的状态信息 + */ + private boolean locked; + private long tryUnlockTimes; + private long lastLockTimestamp; + + private boolean droped; + private long lastPullTimestamp; + private long lastConsumeTimestamp; + + + public long getCommitOffset() { + return commitOffset; + } + + + public void setCommitOffset(long commitOffset) { + this.commitOffset = commitOffset; + } + + + public long getCachedMsgMinOffset() { + return cachedMsgMinOffset; + } + + + public void setCachedMsgMinOffset(long cachedMsgMinOffset) { + this.cachedMsgMinOffset = cachedMsgMinOffset; + } + + + public long getCachedMsgMaxOffset() { + return cachedMsgMaxOffset; + } + + + public void setCachedMsgMaxOffset(long cachedMsgMaxOffset) { + this.cachedMsgMaxOffset = cachedMsgMaxOffset; + } + + + public int getCachedMsgCount() { + return cachedMsgCount; + } + + + public void setCachedMsgCount(int cachedMsgCount) { + this.cachedMsgCount = cachedMsgCount; + } + + + public long getTransactionMsgMinOffset() { + return transactionMsgMinOffset; + } + + + public void setTransactionMsgMinOffset(long transactionMsgMinOffset) { + this.transactionMsgMinOffset = transactionMsgMinOffset; + } + + + public long getTransactionMsgMaxOffset() { + return transactionMsgMaxOffset; + } + + + public void setTransactionMsgMaxOffset(long transactionMsgMaxOffset) { + this.transactionMsgMaxOffset = transactionMsgMaxOffset; + } + + + public int getTransactionMsgCount() { + return transactionMsgCount; + } + + + public void setTransactionMsgCount(int transactionMsgCount) { + this.transactionMsgCount = transactionMsgCount; + } + + + public boolean isLocked() { + return locked; + } + + + public void setLocked(boolean locked) { + this.locked = locked; + } + + + public long getTryUnlockTimes() { + return tryUnlockTimes; + } + + + public void setTryUnlockTimes(long tryUnlockTimes) { + this.tryUnlockTimes = tryUnlockTimes; + } + + + public long getLastLockTimestamp() { + return lastLockTimestamp; + } + + + public void setLastLockTimestamp(long lastLockTimestamp) { + this.lastLockTimestamp = lastLockTimestamp; + } + + + public boolean isDroped() { + return droped; + } + + + public void setDroped(boolean droped) { + this.droped = droped; + } + + + public long getLastPullTimestamp() { + return lastPullTimestamp; + } + + + public void setLastPullTimestamp(long lastPullTimestamp) { + this.lastPullTimestamp = lastPullTimestamp; + } + + + public long getLastConsumeTimestamp() { + return lastConsumeTimestamp; + } + + + public void setLastConsumeTimestamp(long lastConsumeTimestamp) { + this.lastConsumeTimestamp = lastConsumeTimestamp; + } + + + @Override + public String toString() { + return "ProcessQueueInfo [commitOffset=" + commitOffset + ", cachedMsgMinOffset=" + + cachedMsgMinOffset + ", cachedMsgMaxOffset=" + cachedMsgMaxOffset + ", cachedMsgCount=" + + cachedMsgCount + ", transactionMsgMinOffset=" + transactionMsgMinOffset + + ", transactionMsgMaxOffset=" + transactionMsgMaxOffset + ", transactionMsgCount=" + + transactionMsgCount + ", locked=" + locked + ", tryUnlockTimes=" + tryUnlockTimes + + ", lastLockTimestamp=" + UtilAll.timeMillisToHumanString(lastLockTimestamp) + ", droped=" + + droped + ", lastPullTimestamp=" + UtilAll.timeMillisToHumanString(lastPullTimestamp) + + ", lastConsumeTimestamp=" + UtilAll.timeMillisToHumanString(lastConsumeTimestamp) + "]"; + + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ProducerConnection.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ProducerConnection.java new file mode 100644 index 000000000..98fd9a7cf --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ProducerConnection.java @@ -0,0 +1,26 @@ +package com.alibaba.rocketmq.common.protocol.body; + +import java.util.HashSet; + +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; + + +/** + * TODO + * + * @author shijia.wxr + * @since 13-8-5 + */ +public class ProducerConnection extends RemotingSerializable { + private HashSet connectionSet = new HashSet(); + + + public HashSet getConnectionSet() { + return connectionSet; + } + + + public void setConnectionSet(HashSet connectionSet) { + this.connectionSet = connectionSet; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/QueryConsumeTimeSpanBody.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/QueryConsumeTimeSpanBody.java new file mode 100644 index 000000000..1924e1e73 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/QueryConsumeTimeSpanBody.java @@ -0,0 +1,27 @@ +package com.alibaba.rocketmq.common.protocol.body; + +import java.util.HashSet; +import java.util.Set; + +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; + + +/** + * 查看客户端消费组的消费情况。 + * + * @author: manhong.yqd + * @since: 13-12-30 + */ +public class QueryConsumeTimeSpanBody extends RemotingSerializable { + Set consumeTimeSpanSet = new HashSet(); + + + public Set getConsumeTimeSpanSet() { + return consumeTimeSpanSet; + } + + + public void setConsumeTimeSpanSet(Set consumeTimeSpanSet) { + this.consumeTimeSpanSet = consumeTimeSpanSet; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/QueryCorrectionOffsetBody.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/QueryCorrectionOffsetBody.java new file mode 100644 index 000000000..f54ac5211 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/QueryCorrectionOffsetBody.java @@ -0,0 +1,40 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common.protocol.body; + +import java.util.HashMap; +import java.util.Map; + +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; + + +/** + * @author: manhong.yqd + * @since: 14-08-06 + */ +public class QueryCorrectionOffsetBody extends RemotingSerializable { + private Map correctionOffsets = new HashMap(); + + + public Map getCorrectionOffsets() { + return correctionOffsets; + } + + + public void setCorrectionOffsets(Map correctionOffsets) { + this.correctionOffsets = correctionOffsets; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/QueueTimeSpan.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/QueueTimeSpan.java new file mode 100644 index 000000000..b32b03d65 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/QueueTimeSpan.java @@ -0,0 +1,75 @@ +package com.alibaba.rocketmq.common.protocol.body; + +import java.util.Date; + +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.message.MessageQueue; + + +/** + * description + * + * @author: manhong.yqd + * @since: 14-5-28 + */ +public class QueueTimeSpan { + private MessageQueue messageQueue; + private long minTimeStamp; + private long maxTimeStamp; + private long consumeTimeStamp; + + + public MessageQueue getMessageQueue() { + return messageQueue; + } + + + public void setMessageQueue(MessageQueue messageQueue) { + this.messageQueue = messageQueue; + } + + + public long getMinTimeStamp() { + return minTimeStamp; + } + + + public void setMinTimeStamp(long minTimeStamp) { + this.minTimeStamp = minTimeStamp; + } + + + public long getMaxTimeStamp() { + return maxTimeStamp; + } + + + public void setMaxTimeStamp(long maxTimeStamp) { + this.maxTimeStamp = maxTimeStamp; + } + + + public long getConsumeTimeStamp() { + return consumeTimeStamp; + } + + + public void setConsumeTimeStamp(long consumeTimeStamp) { + this.consumeTimeStamp = consumeTimeStamp; + } + + + public String getMinTimeStampStr() { + return UtilAll.formatDate(new Date(minTimeStamp), UtilAll.yyyy_MM_dd_HH_mm_ss_SSS); + } + + + public String getMaxTimeStampStr() { + return UtilAll.formatDate(new Date(maxTimeStamp), UtilAll.yyyy_MM_dd_HH_mm_ss_SSS); + } + + + public String getConsumeTimeStampStr() { + return UtilAll.formatDate(new Date(consumeTimeStamp), UtilAll.yyyy_MM_dd_HH_mm_ss_SSS); + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/RegisterBrokerBody.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/RegisterBrokerBody.java new file mode 100644 index 000000000..cf1bdd3ae --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/RegisterBrokerBody.java @@ -0,0 +1,32 @@ +package com.alibaba.rocketmq.common.protocol.body; + +import java.util.ArrayList; +import java.util.List; + +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; + + +public class RegisterBrokerBody extends RemotingSerializable { + private TopicConfigSerializeWrapper topicConfigSerializeWrapper = new TopicConfigSerializeWrapper(); + private List filterServerList = new ArrayList(); + + + public TopicConfigSerializeWrapper getTopicConfigSerializeWrapper() { + return topicConfigSerializeWrapper; + } + + + public void setTopicConfigSerializeWrapper(TopicConfigSerializeWrapper topicConfigSerializeWrapper) { + this.topicConfigSerializeWrapper = topicConfigSerializeWrapper; + } + + + public List getFilterServerList() { + return filterServerList; + } + + + public void setFilterServerList(List filterServerList) { + this.filterServerList = filterServerList; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ResetOffsetBody.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ResetOffsetBody.java new file mode 100644 index 000000000..a1c470344 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/ResetOffsetBody.java @@ -0,0 +1,27 @@ +package com.alibaba.rocketmq.common.protocol.body; + +import java.util.Map; + +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; + + +/** + * 重置 offset 处理结果。 + * + * @author: manhong.yqd + * @since: 13-12-30 + */ +public class ResetOffsetBody extends RemotingSerializable { + private Map offsetTable; + + + public Map getOffsetTable() { + return offsetTable; + } + + + public void setOffsetTable(Map offsetTable) { + this.offsetTable = offsetTable; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/SubscriptionGroupWrapper.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/SubscriptionGroupWrapper.java new file mode 100644 index 000000000..1cd3c3cf6 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/SubscriptionGroupWrapper.java @@ -0,0 +1,41 @@ +package com.alibaba.rocketmq.common.protocol.body; + +import java.util.concurrent.ConcurrentHashMap; + +import com.alibaba.rocketmq.common.DataVersion; +import com.alibaba.rocketmq.common.subscription.SubscriptionGroupConfig; +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; + + +/** + * 订阅组配置，序列化包装 + * + * @author manhong.yqd + * @since 2013-8-19 + */ +public class SubscriptionGroupWrapper extends RemotingSerializable { + private ConcurrentHashMap subscriptionGroupTable = + new ConcurrentHashMap(1024); + private DataVersion dataVersion = new DataVersion(); + + + public ConcurrentHashMap getSubscriptionGroupTable() { + return subscriptionGroupTable; + } + + + public void setSubscriptionGroupTable( + ConcurrentHashMap subscriptionGroupTable) { + this.subscriptionGroupTable = subscriptionGroupTable; + } + + + public DataVersion getDataVersion() { + return dataVersion; + } + + + public void setDataVersion(DataVersion dataVersion) { + this.dataVersion = dataVersion; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/TopicConfigSerializeWrapper.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/TopicConfigSerializeWrapper.java new file mode 100644 index 000000000..0892068aa --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/TopicConfigSerializeWrapper.java @@ -0,0 +1,34 @@ +package com.alibaba.rocketmq.common.protocol.body; + +import java.util.concurrent.ConcurrentHashMap; + +import com.alibaba.rocketmq.common.DataVersion; +import com.alibaba.rocketmq.common.TopicConfig; +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; + + +public class TopicConfigSerializeWrapper extends RemotingSerializable { + private ConcurrentHashMap topicConfigTable = + new ConcurrentHashMap(); + private DataVersion dataVersion = new DataVersion(); + + + public ConcurrentHashMap getTopicConfigTable() { + return topicConfigTable; + } + + + public void setTopicConfigTable(ConcurrentHashMap topicConfigTable) { + this.topicConfigTable = topicConfigTable; + } + + + public DataVersion getDataVersion() { + return dataVersion; + } + + + public void setDataVersion(DataVersion dataVersion) { + this.dataVersion = dataVersion; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/TopicList.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/TopicList.java new file mode 100644 index 000000000..9d8a092ab --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/TopicList.java @@ -0,0 +1,51 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common.protocol.body; + +import java.util.HashSet; +import java.util.Set; + +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; + + +/** + * @author shijia.wxr + * @since 2013-8-10 + */ +public class TopicList extends RemotingSerializable { + private Set topicList = new HashSet(); + private String brokerAddr; + + + public Set getTopicList() { + return topicList; + } + + + public void setTopicList(Set topicList) { + this.topicList = topicList; + } + + + public String getBrokerAddr() { + return brokerAddr; + } + + + public void setBrokerAddr(String brokerAddr) { + this.brokerAddr = brokerAddr; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/UnlockBatchRequestBody.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/UnlockBatchRequestBody.java new file mode 100644 index 000000000..23960cf94 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/body/UnlockBatchRequestBody.java @@ -0,0 +1,48 @@ +package com.alibaba.rocketmq.common.protocol.body; + +import java.util.HashSet; +import java.util.Set; + +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; + + +/** + * @author shijia.wxr + * @since 2013-6-26 + */ +public class UnlockBatchRequestBody extends RemotingSerializable { + private String consumerGroup; + private String clientId; + private Set mqSet = new HashSet(); + + + public String getConsumerGroup() { + return consumerGroup; + } + + + public void setConsumerGroup(String consumerGroup) { + this.consumerGroup = consumerGroup; + } + + + public String getClientId() { + return clientId; + } + + + public void setClientId(String clientId) { + this.clientId = clientId; + } + + + public Set getMqSet() { + return mqSet; + } + + + public void setMqSet(Set mqSet) { + this.mqSet = mqSet; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/CheckTransactionStateRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/CheckTransactionStateRequestHeader.java new file mode 100644 index 000000000..705cb7aa5 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/CheckTransactionStateRequestHeader.java @@ -0,0 +1,44 @@ +/** + * $Id: EndTransactionRequestHeader.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class CheckTransactionStateRequestHeader implements CommandCustomHeader { + @CFNotNull + private Long tranStateTableOffset; + @CFNotNull + private Long commitLogOffset; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public Long getTranStateTableOffset() { + return tranStateTableOffset; + } + + + public void setTranStateTableOffset(Long tranStateTableOffset) { + this.tranStateTableOffset = tranStateTableOffset; + } + + + public Long getCommitLogOffset() { + return commitLogOffset; + } + + + public void setCommitLogOffset(Long commitLogOffset) { + this.commitLogOffset = commitLogOffset; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/CheckTransactionStateResponseHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/CheckTransactionStateResponseHeader.java new file mode 100644 index 000000000..1aca4635f --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/CheckTransactionStateResponseHeader.java @@ -0,0 +1,80 @@ +/** + * $Id: EndTransactionResponseHeader.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.common.sysflag.MessageSysFlag; +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class CheckTransactionStateResponseHeader implements CommandCustomHeader { + @CFNotNull + private String producerGroup; + @CFNotNull + private Long tranStateTableOffset; + @CFNotNull + private Long commitLogOffset; + @CFNotNull + private Integer commitOrRollback; // TransactionCommitType + + + // TransactionRollbackType + + @Override + public void checkFields() throws RemotingCommandException { + if (MessageSysFlag.TransactionCommitType == this.commitOrRollback) { + return; + } + + if (MessageSysFlag.TransactionRollbackType == this.commitOrRollback) { + return; + } + + throw new RemotingCommandException("commitOrRollback field wrong"); + } + + + public String getProducerGroup() { + return producerGroup; + } + + + public void setProducerGroup(String producerGroup) { + this.producerGroup = producerGroup; + } + + + public Long getTranStateTableOffset() { + return tranStateTableOffset; + } + + + public void setTranStateTableOffset(Long tranStateTableOffset) { + this.tranStateTableOffset = tranStateTableOffset; + } + + + public Long getCommitLogOffset() { + return commitLogOffset; + } + + + public void setCommitLogOffset(Long commitLogOffset) { + this.commitLogOffset = commitLogOffset; + } + + + public Integer getCommitOrRollback() { + return commitOrRollback; + } + + + public void setCommitOrRollback(Integer commitOrRollback) { + this.commitOrRollback = commitOrRollback; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/CloneGroupOffsetRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/CloneGroupOffsetRequestHeader.java new file mode 100644 index 000000000..52fae5ec8 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/CloneGroupOffsetRequestHeader.java @@ -0,0 +1,68 @@ +/** + * $Id: DeleteTopicRequestHeader.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author manhong.yqd + * @since 14-09-15 + */ +public class CloneGroupOffsetRequestHeader implements CommandCustomHeader { + @CFNotNull + private String srcGroup; + @CFNotNull + private String destGroup; + private String topic; + private boolean offline; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public String getDestGroup() { + return destGroup; + } + + + public void setDestGroup(String destGroup) { + this.destGroup = destGroup; + } + + + public String getTopic() { + return topic; + } + + + public void setTopic(String topic) { + this.topic = topic; + } + + + public String getSrcGroup() { + + return srcGroup; + } + + + public void setSrcGroup(String srcGroup) { + this.srcGroup = srcGroup; + } + + + public boolean isOffline() { + return offline; + } + + + public void setOffline(boolean offline) { + this.offline = offline; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/ConsumeMessageDirectlyResultRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/ConsumeMessageDirectlyResultRequestHeader.java new file mode 100644 index 000000000..60263abbb --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/ConsumeMessageDirectlyResultRequestHeader.java @@ -0,0 +1,63 @@ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.annotation.CFNullable; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +public class ConsumeMessageDirectlyResultRequestHeader implements CommandCustomHeader { + @CFNotNull + private String consumerGroup; + @CFNullable + private String clientId; + @CFNullable + private String msgId; + @CFNullable + private String brokerName; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public String getConsumerGroup() { + return consumerGroup; + } + + + public void setConsumerGroup(String consumerGroup) { + this.consumerGroup = consumerGroup; + } + + + public String getBrokerName() { + return brokerName; + } + + + public void setBrokerName(String brokerName) { + this.brokerName = brokerName; + } + + + public String getClientId() { + return clientId; + } + + + public void setClientId(String clientId) { + this.clientId = clientId; + } + + + public String getMsgId() { + return msgId; + } + + + public void setMsgId(String msgId) { + this.msgId = msgId; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/ConsumerSendMsgBackRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/ConsumerSendMsgBackRequestHeader.java new file mode 100644 index 000000000..ad9832add --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/ConsumerSendMsgBackRequestHeader.java @@ -0,0 +1,98 @@ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.annotation.CFNullable; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + * @since 2013-6-18 + */ +public class ConsumerSendMsgBackRequestHeader implements CommandCustomHeader { + @CFNotNull + private Long offset; + @CFNotNull + private String group; + @CFNotNull + private Integer delayLevel; + private String originMsgId; + private String originTopic; + @CFNullable + private boolean unitMode = false; + + + @Override + public void checkFields() throws RemotingCommandException { + + } + + + public Long getOffset() { + return offset; + } + + + public void setOffset(Long offset) { + this.offset = offset; + } + + + public String getGroup() { + return group; + } + + + public void setGroup(String group) { + this.group = group; + } + + + public Integer getDelayLevel() { + return delayLevel; + } + + + public void setDelayLevel(Integer delayLevel) { + this.delayLevel = delayLevel; + } + + + public String getOriginMsgId() { + return originMsgId; + } + + + public void setOriginMsgId(String originMsgId) { + this.originMsgId = originMsgId; + } + + + public String getOriginTopic() { + return originTopic; + } + + + public void setOriginTopic(String originTopic) { + this.originTopic = originTopic; + } + + + public boolean isUnitMode() { + return unitMode; + } + + + public void setUnitMode(boolean unitMode) { + this.unitMode = unitMode; + } + + + @Override + public String toString() { + return "ConsumerSendMsgBackRequestHeader [group=" + group + ", originTopic=" + originTopic + + ", originMsgId=" + originMsgId + ", delayLevel=" + delayLevel + ", unitMode=" + unitMode + + "]"; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/CreateTopicRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/CreateTopicRequestHeader.java new file mode 100644 index 000000000..44c457321 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/CreateTopicRequestHeader.java @@ -0,0 +1,127 @@ +/** + * $Id: CreateTopicRequestHeader.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.common.TopicFilterType; +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class CreateTopicRequestHeader implements CommandCustomHeader { + @CFNotNull + private String topic; + @CFNotNull + private String defaultTopic; + @CFNotNull + private Integer readQueueNums; + @CFNotNull + private Integer writeQueueNums; + @CFNotNull + private Integer perm; + @CFNotNull + private String topicFilterType; + private Integer topicSysFlag; + @CFNotNull + private Boolean order = false; + + + @Override + public void checkFields() throws RemotingCommandException { + try { + TopicFilterType.valueOf(this.topicFilterType); + } + catch (Exception e) { + throw new RemotingCommandException("topicFilterType = [" + topicFilterType + "] value invalid", e); + } + } + + + public TopicFilterType getTopicFilterTypeEnum() { + return TopicFilterType.valueOf(this.topicFilterType); + } + + + public String getTopic() { + return topic; + } + + + public void setTopic(String topic) { + this.topic = topic; + } + + + public String getDefaultTopic() { + return defaultTopic; + } + + + public void setDefaultTopic(String defaultTopic) { + this.defaultTopic = defaultTopic; + } + + + public Integer getReadQueueNums() { + return readQueueNums; + } + + + public void setReadQueueNums(Integer readQueueNums) { + this.readQueueNums = readQueueNums; + } + + + public Integer getWriteQueueNums() { + return writeQueueNums; + } + + + public void setWriteQueueNums(Integer writeQueueNums) { + this.writeQueueNums = writeQueueNums; + } + + + public Integer getPerm() { + return perm; + } + + + public void setPerm(Integer perm) { + this.perm = perm; + } + + + public String getTopicFilterType() { + return topicFilterType; + } + + + public void setTopicFilterType(String topicFilterType) { + this.topicFilterType = topicFilterType; + } + + + public Integer getTopicSysFlag() { + return topicSysFlag; + } + + + public void setTopicSysFlag(Integer topicSysFlag) { + this.topicSysFlag = topicSysFlag; + } + + + public Boolean getOrder() { + return order; + } + + + public void setOrder(Boolean order) { + this.order = order; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/DeleteSubscriptionGroupRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/DeleteSubscriptionGroupRequestHeader.java new file mode 100644 index 000000000..00b15cf6e --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/DeleteSubscriptionGroupRequestHeader.java @@ -0,0 +1,32 @@ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * 删除订阅组请求参数 + * + * @author manhong.yqd + * @since 2013-8-22 + */ +public class DeleteSubscriptionGroupRequestHeader implements CommandCustomHeader { + @CFNotNull + private String groupName; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public String getGroupName() { + return groupName; + } + + + public void setGroupName(String groupName) { + this.groupName = groupName; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/DeleteTopicRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/DeleteTopicRequestHeader.java new file mode 100644 index 000000000..3d2098455 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/DeleteTopicRequestHeader.java @@ -0,0 +1,32 @@ +/** + * $Id: DeleteTopicRequestHeader.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class DeleteTopicRequestHeader implements CommandCustomHeader { + @CFNotNull + private String topic; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public String getTopic() { + return topic; + } + + + public void setTopic(String topic) { + this.topic = topic; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/EndTransactionRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/EndTransactionRequestHeader.java new file mode 100644 index 000000000..4103f918f --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/EndTransactionRequestHeader.java @@ -0,0 +1,120 @@ +/** + * $Id: EndTransactionRequestHeader.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.common.sysflag.MessageSysFlag; +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.annotation.CFNullable; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class EndTransactionRequestHeader implements CommandCustomHeader { + @CFNotNull + private String producerGroup; + @CFNotNull + private Long tranStateTableOffset; + @CFNotNull + private Long commitLogOffset; + @CFNotNull + private Integer commitOrRollback; // TransactionCommitType + // TransactionRollbackType + // TransactionNotType + + @CFNullable + private Boolean fromTransactionCheck = false; + + @CFNotNull + private String msgId; + + + @Override + public void checkFields() throws RemotingCommandException { + if (MessageSysFlag.TransactionNotType == this.commitOrRollback) { + return; + } + + if (MessageSysFlag.TransactionCommitType == this.commitOrRollback) { + return; + } + + if (MessageSysFlag.TransactionRollbackType == this.commitOrRollback) { + return; + } + + throw new RemotingCommandException("commitOrRollback field wrong"); + } + + + public String getProducerGroup() { + return producerGroup; + } + + + public void setProducerGroup(String producerGroup) { + this.producerGroup = producerGroup; + } + + + public Long getTranStateTableOffset() { + return tranStateTableOffset; + } + + + public void setTranStateTableOffset(Long tranStateTableOffset) { + this.tranStateTableOffset = tranStateTableOffset; + } + + + public Long getCommitLogOffset() { + return commitLogOffset; + } + + + public void setCommitLogOffset(Long commitLogOffset) { + this.commitLogOffset = commitLogOffset; + } + + + public Integer getCommitOrRollback() { + return commitOrRollback; + } + + + public void setCommitOrRollback(Integer commitOrRollback) { + this.commitOrRollback = commitOrRollback; + } + + + public Boolean getFromTransactionCheck() { + return fromTransactionCheck; + } + + + public void setFromTransactionCheck(Boolean fromTransactionCheck) { + this.fromTransactionCheck = fromTransactionCheck; + } + + + public String getMsgId() { + return msgId; + } + + + public void setMsgId(String msgId) { + this.msgId = msgId; + } + + + @Override + public String toString() { + return "EndTransactionRequestHeader [producerGroup=" + producerGroup + ", tranStateTableOffset=" + + tranStateTableOffset + ", commitLogOffset=" + commitLogOffset + ", commitOrRollback=" + + commitOrRollback + ", fromTransactionCheck=" + fromTransactionCheck + ", msgId=" + msgId + + "]"; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/EndTransactionResponseHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/EndTransactionResponseHeader.java new file mode 100644 index 000000000..3b0f772ef --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/EndTransactionResponseHeader.java @@ -0,0 +1,21 @@ +/** + * $Id: EndTransactionResponseHeader.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class EndTransactionResponseHeader implements CommandCustomHeader { + + @Override + public void checkFields() throws RemotingCommandException { + // TODO Auto-generated method stub + + } + +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetAllTopicConfigResponseHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetAllTopicConfigResponseHeader.java new file mode 100644 index 000000000..d9ba1a52b --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetAllTopicConfigResponseHeader.java @@ -0,0 +1,18 @@ +/** + * $Id: GetAllTopicConfigResponseHeader.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class GetAllTopicConfigResponseHeader implements CommandCustomHeader { + + @Override + public void checkFields() throws RemotingCommandException { + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetBrokerConfigResponseHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetBrokerConfigResponseHeader.java new file mode 100644 index 000000000..bb9f33270 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetBrokerConfigResponseHeader.java @@ -0,0 +1,32 @@ +/** + * $Id: GetBrokerConfigResponseHeader.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class GetBrokerConfigResponseHeader implements CommandCustomHeader { + @CFNotNull + private String version; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public String getVersion() { + return version; + } + + + public void setVersion(String version) { + this.version = version; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetConsumeStatsRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetConsumeStatsRequestHeader.java new file mode 100644 index 000000000..1b6972f13 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetConsumeStatsRequestHeader.java @@ -0,0 +1,47 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + * @since 2013-8-11 + */ +public class GetConsumeStatsRequestHeader implements CommandCustomHeader { + @CFNotNull + private String consumerGroup; + + + @Override + public void checkFields() throws RemotingCommandException { + // TODO Auto-generated method stub + + } + + + public String getConsumerGroup() { + return consumerGroup; + } + + + public void setConsumerGroup(String consumerGroup) { + this.consumerGroup = consumerGroup; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetConsumerConnectionListRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetConsumerConnectionListRequestHeader.java new file mode 100644 index 000000000..b0542ffc0 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetConsumerConnectionListRequestHeader.java @@ -0,0 +1,34 @@ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * TODO + * + * @author shijia.wxr + * @since 13-8-5 + */ +public class GetConsumerConnectionListRequestHeader implements CommandCustomHeader { + @CFNotNull + private String consumerGroup; + + + @Override + public void checkFields() throws RemotingCommandException { + // To change body of implemented methods use File | Settings | File + // Templates. + } + + + public String getConsumerGroup() { + return consumerGroup; + } + + + public void setConsumerGroup(String consumerGroup) { + this.consumerGroup = consumerGroup; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetConsumerListByGroupRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetConsumerListByGroupRequestHeader.java new file mode 100644 index 000000000..7bdddd0e6 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetConsumerListByGroupRequestHeader.java @@ -0,0 +1,29 @@ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class GetConsumerListByGroupRequestHeader implements CommandCustomHeader { + @CFNotNull + private String consumerGroup; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public String getConsumerGroup() { + return consumerGroup; + } + + + public void setConsumerGroup(String consumerGroup) { + this.consumerGroup = consumerGroup; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetConsumerListByGroupResponseBody.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetConsumerListByGroupResponseBody.java new file mode 100644 index 000000000..2e375118c --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetConsumerListByGroupResponseBody.java @@ -0,0 +1,23 @@ +package com.alibaba.rocketmq.common.protocol.header; + +import java.util.List; + +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; + + +/** + * @author shijia.wxr + */ +public class GetConsumerListByGroupResponseBody extends RemotingSerializable { + private List consumerIdList; + + + public List getConsumerIdList() { + return consumerIdList; + } + + + public void setConsumerIdList(List consumerIdList) { + this.consumerIdList = consumerIdList; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetConsumerListByGroupResponseHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetConsumerListByGroupResponseHeader.java new file mode 100644 index 000000000..2e4b30cb0 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetConsumerListByGroupResponseHeader.java @@ -0,0 +1,15 @@ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class GetConsumerListByGroupResponseHeader implements CommandCustomHeader { + + @Override + public void checkFields() throws RemotingCommandException { + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetConsumerRunningInfoRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetConsumerRunningInfoRequestHeader.java new file mode 100644 index 000000000..4f01dc8d8 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetConsumerRunningInfoRequestHeader.java @@ -0,0 +1,54 @@ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.annotation.CFNullable; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class GetConsumerRunningInfoRequestHeader implements CommandCustomHeader { + @CFNotNull + private String consumerGroup; + @CFNotNull + private String clientId; + @CFNullable + private boolean jstackEnable; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public String getConsumerGroup() { + return consumerGroup; + } + + + public void setConsumerGroup(String consumerGroup) { + this.consumerGroup = consumerGroup; + } + + + public String getClientId() { + return clientId; + } + + + public void setClientId(String clientId) { + this.clientId = clientId; + } + + + public boolean isJstackEnable() { + return jstackEnable; + } + + + public void setJstackEnable(boolean jstackEnable) { + this.jstackEnable = jstackEnable; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetConsumerStatusRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetConsumerStatusRequestHeader.java new file mode 100644 index 000000000..b80a6f7a2 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetConsumerStatusRequestHeader.java @@ -0,0 +1,58 @@ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.annotation.CFNullable; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * 查看客户端消费组的消费情况。 + * + * @author: manhong.yqd + * @since: 13-12-30 + */ +public class GetConsumerStatusRequestHeader implements CommandCustomHeader { + @CFNotNull + private String topic; + @CFNotNull + private String group; + @CFNullable + private String clientAddr; + + + @Override + public void checkFields() throws RemotingCommandException { + // TODO Auto-generated method stub + } + + + public String getTopic() { + return topic; + } + + + public void setTopic(String topic) { + this.topic = topic; + } + + + public String getGroup() { + return group; + } + + + public void setGroup(String group) { + this.group = group; + } + + + public String getClientAddr() { + return clientAddr; + } + + + public void setClientAddr(String clientAddr) { + this.clientAddr = clientAddr; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetEarliestMsgStoretimeRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetEarliestMsgStoretimeRequestHeader.java new file mode 100644 index 000000000..984004655 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetEarliestMsgStoretimeRequestHeader.java @@ -0,0 +1,45 @@ +/** + * $Id: GetEarliestMsgStoretimeRequestHeader.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class GetEarliestMsgStoretimeRequestHeader implements CommandCustomHeader { + @CFNotNull + private String topic; + @CFNotNull + private Integer queueId; + + + @Override + public void checkFields() throws RemotingCommandException { + // TODO Auto-generated method stub + } + + + public String getTopic() { + return topic; + } + + + public void setTopic(String topic) { + this.topic = topic; + } + + + public Integer getQueueId() { + return queueId; + } + + + public void setQueueId(Integer queueId) { + this.queueId = queueId; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetEarliestMsgStoretimeResponseHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetEarliestMsgStoretimeResponseHeader.java new file mode 100644 index 000000000..185349d54 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetEarliestMsgStoretimeResponseHeader.java @@ -0,0 +1,32 @@ +/** + * $Id: GetEarliestMsgStoretimeResponseHeader.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class GetEarliestMsgStoretimeResponseHeader implements CommandCustomHeader { + @CFNotNull + private Long timestamp; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public Long getTimestamp() { + return timestamp; + } + + + public void setTimestamp(Long timestamp) { + this.timestamp = timestamp; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetMaxOffsetRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetMaxOffsetRequestHeader.java new file mode 100644 index 000000000..c96c77348 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetMaxOffsetRequestHeader.java @@ -0,0 +1,45 @@ +/** + * $Id: GetMaxOffsetRequestHeader.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class GetMaxOffsetRequestHeader implements CommandCustomHeader { + @CFNotNull + private String topic; + @CFNotNull + private Integer queueId; + + + @Override + public void checkFields() throws RemotingCommandException { + // TODO Auto-generated method stub + } + + + public String getTopic() { + return topic; + } + + + public void setTopic(String topic) { + this.topic = topic; + } + + + public Integer getQueueId() { + return queueId; + } + + + public void setQueueId(Integer queueId) { + this.queueId = queueId; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetMaxOffsetResponseHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetMaxOffsetResponseHeader.java new file mode 100644 index 000000000..c5ec5499d --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetMaxOffsetResponseHeader.java @@ -0,0 +1,32 @@ +/** + * $Id: GetMaxOffsetResponseHeader.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class GetMaxOffsetResponseHeader implements CommandCustomHeader { + @CFNotNull + private Long offset; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public Long getOffset() { + return offset; + } + + + public void setOffset(Long offset) { + this.offset = offset; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetMinOffsetRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetMinOffsetRequestHeader.java new file mode 100644 index 000000000..254a9bc45 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetMinOffsetRequestHeader.java @@ -0,0 +1,45 @@ +/** + * $Id: GetMinOffsetRequestHeader.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class GetMinOffsetRequestHeader implements CommandCustomHeader { + @CFNotNull + private String topic; + @CFNotNull + private Integer queueId; + + + @Override + public void checkFields() throws RemotingCommandException { + // TODO Auto-generated method stub + } + + + public String getTopic() { + return topic; + } + + + public void setTopic(String topic) { + this.topic = topic; + } + + + public Integer getQueueId() { + return queueId; + } + + + public void setQueueId(Integer queueId) { + this.queueId = queueId; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetMinOffsetResponseHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetMinOffsetResponseHeader.java new file mode 100644 index 000000000..e8ae3db81 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetMinOffsetResponseHeader.java @@ -0,0 +1,32 @@ +/** + * $Id: GetMinOffsetResponseHeader.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class GetMinOffsetResponseHeader implements CommandCustomHeader { + @CFNotNull + private Long offset; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public Long getOffset() { + return offset; + } + + + public void setOffset(Long offset) { + this.offset = offset; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetProducerConnectionListRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetProducerConnectionListRequestHeader.java new file mode 100644 index 000000000..4a8e98a7d --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetProducerConnectionListRequestHeader.java @@ -0,0 +1,34 @@ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * TODO + * + * @author shijia.wxr + * @since 13-8-5 + */ +public class GetProducerConnectionListRequestHeader implements CommandCustomHeader { + @CFNotNull + private String producerGroup; + + + @Override + public void checkFields() throws RemotingCommandException { + // To change body of implemented methods use File | Settings | File + // Templates. + } + + + public String getProducerGroup() { + return producerGroup; + } + + + public void setProducerGroup(String producerGroup) { + this.producerGroup = producerGroup; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetTopicStatsInfoRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetTopicStatsInfoRequestHeader.java new file mode 100644 index 000000000..b470d6827 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetTopicStatsInfoRequestHeader.java @@ -0,0 +1,30 @@ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + * @since 13-8-4 + */ +public class GetTopicStatsInfoRequestHeader implements CommandCustomHeader { + @CFNotNull + private String topic; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public String getTopic() { + return topic; + } + + + public void setTopic(String topic) { + this.topic = topic; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetTopicsByClusterRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetTopicsByClusterRequestHeader.java new file mode 100644 index 000000000..ae03f0d27 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/GetTopicsByClusterRequestHeader.java @@ -0,0 +1,30 @@ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author manhong.yqd + * @since 14-03-05 + */ +public class GetTopicsByClusterRequestHeader implements CommandCustomHeader { + @CFNotNull + private String cluster; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public String getCluster() { + return cluster; + } + + + public void setCluster(String cluster) { + this.cluster = cluster; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/NotifyConsumerIdsChangedRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/NotifyConsumerIdsChangedRequestHeader.java new file mode 100644 index 000000000..89e561b90 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/NotifyConsumerIdsChangedRequestHeader.java @@ -0,0 +1,30 @@ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class NotifyConsumerIdsChangedRequestHeader implements CommandCustomHeader { + @CFNotNull + private String consumerGroup; + + + @Override + public void checkFields() throws RemotingCommandException { + // TODO Auto-generated method stub + } + + + public String getConsumerGroup() { + return consumerGroup; + } + + + public void setConsumerGroup(String consumerGroup) { + this.consumerGroup = consumerGroup; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/PullMessageRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/PullMessageRequestHeader.java new file mode 100644 index 000000000..c60f2eb14 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/PullMessageRequestHeader.java @@ -0,0 +1,141 @@ +/** + * $Id: PullMessageRequestHeader.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.annotation.CFNullable; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class PullMessageRequestHeader implements CommandCustomHeader { + @CFNotNull + private String consumerGroup; + @CFNotNull + private String topic; + @CFNotNull + private Integer queueId; + @CFNotNull + private Long queueOffset; + @CFNotNull + private Integer maxMsgNums; + @CFNotNull + private Integer sysFlag; + @CFNotNull + private Long commitOffset; + @CFNotNull + private Long suspendTimeoutMillis; + @CFNullable + private String subscription; + @CFNotNull + private Long subVersion; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public String getConsumerGroup() { + return consumerGroup; + } + + + public void setConsumerGroup(String consumerGroup) { + this.consumerGroup = consumerGroup; + } + + + public String getTopic() { + return topic; + } + + + public void setTopic(String topic) { + this.topic = topic; + } + + + public Integer getQueueId() { + return queueId; + } + + + public void setQueueId(Integer queueId) { + this.queueId = queueId; + } + + + public Long getQueueOffset() { + return queueOffset; + } + + + public void setQueueOffset(Long queueOffset) { + this.queueOffset = queueOffset; + } + + + public Integer getMaxMsgNums() { + return maxMsgNums; + } + + + public void setMaxMsgNums(Integer maxMsgNums) { + this.maxMsgNums = maxMsgNums; + } + + + public Integer getSysFlag() { + return sysFlag; + } + + + public void setSysFlag(Integer sysFlag) { + this.sysFlag = sysFlag; + } + + + public Long getCommitOffset() { + return commitOffset; + } + + + public void setCommitOffset(Long commitOffset) { + this.commitOffset = commitOffset; + } + + + public Long getSuspendTimeoutMillis() { + return suspendTimeoutMillis; + } + + + public void setSuspendTimeoutMillis(Long suspendTimeoutMillis) { + this.suspendTimeoutMillis = suspendTimeoutMillis; + } + + + public String getSubscription() { + return subscription; + } + + + public void setSubscription(String subscription) { + this.subscription = subscription; + } + + + public Long getSubVersion() { + return subVersion; + } + + + public void setSubVersion(Long subVersion) { + this.subVersion = subVersion; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/PullMessageResponseHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/PullMessageResponseHeader.java new file mode 100644 index 000000000..dee1f13e5 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/PullMessageResponseHeader.java @@ -0,0 +1,68 @@ +/** + * $Id: PullMessageResponseHeader.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class PullMessageResponseHeader implements CommandCustomHeader { + @CFNotNull + private Long suggestWhichBrokerId; + @CFNotNull + private Long nextBeginOffset; + @CFNotNull + private Long minOffset; + @CFNotNull + private Long maxOffset; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public Long getNextBeginOffset() { + return nextBeginOffset; + } + + + public void setNextBeginOffset(Long nextBeginOffset) { + this.nextBeginOffset = nextBeginOffset; + } + + + public Long getMinOffset() { + return minOffset; + } + + + public void setMinOffset(Long minOffset) { + this.minOffset = minOffset; + } + + + public Long getMaxOffset() { + return maxOffset; + } + + + public void setMaxOffset(Long maxOffset) { + this.maxOffset = maxOffset; + } + + + public Long getSuggestWhichBrokerId() { + return suggestWhichBrokerId; + } + + + public void setSuggestWhichBrokerId(Long suggestWhichBrokerId) { + this.suggestWhichBrokerId = suggestWhichBrokerId; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/QueryConsumeTimeSpanRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/QueryConsumeTimeSpanRequestHeader.java new file mode 100644 index 000000000..a631266cc --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/QueryConsumeTimeSpanRequestHeader.java @@ -0,0 +1,44 @@ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * 根据 topic 和 group 获取消息的时间跨度 + * + * @author: manhong.yqd + * @since: 13-12-30 + */ +public class QueryConsumeTimeSpanRequestHeader implements CommandCustomHeader { + @CFNotNull + private String topic; + @CFNotNull + private String group; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public String getTopic() { + return topic; + } + + + public void setTopic(String topic) { + this.topic = topic; + } + + + public String getGroup() { + return group; + } + + + public void setGroup(String group) { + this.group = group; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/QueryConsumerOffsetRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/QueryConsumerOffsetRequestHeader.java new file mode 100644 index 000000000..761bafc9e --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/QueryConsumerOffsetRequestHeader.java @@ -0,0 +1,56 @@ +/** + * $Id: QueryConsumerOffsetRequestHeader.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class QueryConsumerOffsetRequestHeader implements CommandCustomHeader { + @CFNotNull + private String consumerGroup; + @CFNotNull + private String topic; + @CFNotNull + private Integer queueId; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public String getConsumerGroup() { + return consumerGroup; + } + + + public void setConsumerGroup(String consumerGroup) { + this.consumerGroup = consumerGroup; + } + + + public String getTopic() { + return topic; + } + + + public void setTopic(String topic) { + this.topic = topic; + } + + + public Integer getQueueId() { + return queueId; + } + + + public void setQueueId(Integer queueId) { + this.queueId = queueId; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/QueryConsumerOffsetResponseHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/QueryConsumerOffsetResponseHeader.java new file mode 100644 index 000000000..c0b768e4d --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/QueryConsumerOffsetResponseHeader.java @@ -0,0 +1,34 @@ +/** + * $Id: QueryConsumerOffsetResponseHeader.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class QueryConsumerOffsetResponseHeader implements CommandCustomHeader { + @CFNotNull + private Long offset; + + + @Override + public void checkFields() throws RemotingCommandException { + // TODO Auto-generated method stub + + } + + + public Long getOffset() { + return offset; + } + + + public void setOffset(Long offset) { + this.offset = offset; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/QueryCorrectionOffsetHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/QueryCorrectionOffsetHeader.java new file mode 100644 index 000000000..309935a21 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/QueryCorrectionOffsetHeader.java @@ -0,0 +1,59 @@ +/** + * $Id: GetMinOffsetRequestHeader.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * 查找被纠正的 offset + * + * @author: manhong.yqd + * @since: 14-08-06 + */ +public class QueryCorrectionOffsetHeader implements CommandCustomHeader { + private String filterGroups; + @CFNotNull + private String compareGroup; + @CFNotNull + private String topic; + + + @Override + public void checkFields() throws RemotingCommandException { + // TODO Auto-generated method stub + } + + + public String getFilterGroups() { + return filterGroups; + } + + + public void setFilterGroups(String filterGroups) { + this.filterGroups = filterGroups; + } + + + public String getCompareGroup() { + return compareGroup; + } + + + public void setCompareGroup(String compareGroup) { + this.compareGroup = compareGroup; + } + + + public String getTopic() { + return topic; + } + + + public void setTopic(String topic) { + this.topic = topic; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/QueryMessageRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/QueryMessageRequestHeader.java new file mode 100644 index 000000000..957da6d34 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/QueryMessageRequestHeader.java @@ -0,0 +1,81 @@ +/** + * $Id: QueryMessageRequestHeader.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class QueryMessageRequestHeader implements CommandCustomHeader { + @CFNotNull + private String topic; + @CFNotNull + private String key; + @CFNotNull + private Integer maxNum; + @CFNotNull + private Long beginTimestamp; + @CFNotNull + private Long endTimestamp; + + + @Override + public void checkFields() throws RemotingCommandException { + + } + + + public String getTopic() { + return topic; + } + + + public void setTopic(String topic) { + this.topic = topic; + } + + + public String getKey() { + return key; + } + + + public void setKey(String key) { + this.key = key; + } + + + public Integer getMaxNum() { + return maxNum; + } + + + public void setMaxNum(Integer maxNum) { + this.maxNum = maxNum; + } + + + public Long getBeginTimestamp() { + return beginTimestamp; + } + + + public void setBeginTimestamp(Long beginTimestamp) { + this.beginTimestamp = beginTimestamp; + } + + + public Long getEndTimestamp() { + return endTimestamp; + } + + + public void setEndTimestamp(Long endTimestamp) { + this.endTimestamp = endTimestamp; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/QueryMessageResponseHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/QueryMessageResponseHeader.java new file mode 100644 index 000000000..6cb334e53 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/QueryMessageResponseHeader.java @@ -0,0 +1,44 @@ +/** + * $Id: QueryMessageResponseHeader.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class QueryMessageResponseHeader implements CommandCustomHeader { + @CFNotNull + private Long indexLastUpdateTimestamp; + @CFNotNull + private Long indexLastUpdatePhyoffset; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public Long getIndexLastUpdateTimestamp() { + return indexLastUpdateTimestamp; + } + + + public void setIndexLastUpdateTimestamp(Long indexLastUpdateTimestamp) { + this.indexLastUpdateTimestamp = indexLastUpdateTimestamp; + } + + + public Long getIndexLastUpdatePhyoffset() { + return indexLastUpdatePhyoffset; + } + + + public void setIndexLastUpdatePhyoffset(Long indexLastUpdatePhyoffset) { + this.indexLastUpdatePhyoffset = indexLastUpdatePhyoffset; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/QueryTopicConsumeByWhoRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/QueryTopicConsumeByWhoRequestHeader.java new file mode 100644 index 000000000..5832a5472 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/QueryTopicConsumeByWhoRequestHeader.java @@ -0,0 +1,33 @@ +/** + * $Id: QueryMessageRequestHeader.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class QueryTopicConsumeByWhoRequestHeader implements CommandCustomHeader { + @CFNotNull + private String topic; + + + @Override + public void checkFields() throws RemotingCommandException { + + } + + + public String getTopic() { + return topic; + } + + + public void setTopic(String topic) { + this.topic = topic; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/ResetOffsetRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/ResetOffsetRequestHeader.java new file mode 100644 index 000000000..1ff1c3996 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/ResetOffsetRequestHeader.java @@ -0,0 +1,69 @@ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * 重置 offset。 + * + * @author: manhong.yqd + * @since: 13-12-30 + */ +public class ResetOffsetRequestHeader implements CommandCustomHeader { + @CFNotNull + private String topic; + @CFNotNull + private String group; + @CFNotNull + private long timestamp; + @CFNotNull + private boolean isForce; + + + public String getTopic() { + return topic; + } + + + public void setTopic(String topic) { + this.topic = topic; + } + + + public String getGroup() { + return group; + } + + + public void setGroup(String group) { + this.group = group; + } + + + public long getTimestamp() { + return timestamp; + } + + + public void setTimestamp(long timestamp) { + this.timestamp = timestamp; + } + + + public boolean isForce() { + return isForce; + } + + + public void setForce(boolean isForce) { + this.isForce = isForce; + } + + + @Override + public void checkFields() throws RemotingCommandException { + + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/SearchOffsetRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/SearchOffsetRequestHeader.java new file mode 100644 index 000000000..1e5e7c7e2 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/SearchOffsetRequestHeader.java @@ -0,0 +1,59 @@ +/** + * $Id: SearchOffsetRequestHeader.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class SearchOffsetRequestHeader implements CommandCustomHeader { + @CFNotNull + private String topic; + @CFNotNull + private Integer queueId; + @CFNotNull + private Long timestamp; + + + @Override + public void checkFields() throws RemotingCommandException { + // TODO Auto-generated method stub + + } + + + public String getTopic() { + return topic; + } + + + public void setTopic(String topic) { + this.topic = topic; + } + + + public Integer getQueueId() { + return queueId; + } + + + public void setQueueId(Integer queueId) { + this.queueId = queueId; + } + + + public Long getTimestamp() { + return timestamp; + } + + + public void setTimestamp(Long timestamp) { + this.timestamp = timestamp; + } + +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/SearchOffsetResponseHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/SearchOffsetResponseHeader.java new file mode 100644 index 000000000..9d812ee16 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/SearchOffsetResponseHeader.java @@ -0,0 +1,32 @@ +/** + * $Id: SearchOffsetResponseHeader.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class SearchOffsetResponseHeader implements CommandCustomHeader { + @CFNotNull + private Long offset; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public Long getOffset() { + return offset; + } + + + public void setOffset(Long offset) { + this.offset = offset; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/SendMessageRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/SendMessageRequestHeader.java new file mode 100644 index 000000000..b3cd71c3f --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/SendMessageRequestHeader.java @@ -0,0 +1,153 @@ +/** + * $Id: SendMessageRequestHeader.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.annotation.CFNullable; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class SendMessageRequestHeader implements CommandCustomHeader { + @CFNotNull + private String producerGroup; + @CFNotNull + private String topic; + @CFNotNull + private String defaultTopic; + @CFNotNull + private Integer defaultTopicQueueNums; + @CFNotNull + private Integer queueId; + @CFNotNull + private Integer sysFlag; + @CFNotNull + private Long bornTimestamp; + @CFNotNull + private Integer flag; + @CFNullable + private String properties; + @CFNullable + private Integer reconsumeTimes; + @CFNullable + private boolean unitMode = false; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public String getProducerGroup() { + return producerGroup; + } + + + public void setProducerGroup(String producerGroup) { + this.producerGroup = producerGroup; + } + + + public String getTopic() { + return topic; + } + + + public void setTopic(String topic) { + this.topic = topic; + } + + + public String getDefaultTopic() { + return defaultTopic; + } + + + public void setDefaultTopic(String defaultTopic) { + this.defaultTopic = defaultTopic; + } + + + public Integer getDefaultTopicQueueNums() { + return defaultTopicQueueNums; + } + + + public void setDefaultTopicQueueNums(Integer defaultTopicQueueNums) { + this.defaultTopicQueueNums = defaultTopicQueueNums; + } + + + public Integer getQueueId() { + return queueId; + } + + + public void setQueueId(Integer queueId) { + this.queueId = queueId; + } + + + public Integer getSysFlag() { + return sysFlag; + } + + + public void setSysFlag(Integer sysFlag) { + this.sysFlag = sysFlag; + } + + + public Long getBornTimestamp() { + return bornTimestamp; + } + + + public void setBornTimestamp(Long bornTimestamp) { + this.bornTimestamp = bornTimestamp; + } + + + public Integer getFlag() { + return flag; + } + + + public void setFlag(Integer flag) { + this.flag = flag; + } + + + public String getProperties() { + return properties; + } + + + public void setProperties(String properties) { + this.properties = properties; + } + + + public Integer getReconsumeTimes() { + return reconsumeTimes; + } + + + public void setReconsumeTimes(Integer reconsumeTimes) { + this.reconsumeTimes = reconsumeTimes; + } + + + public boolean isUnitMode() { + return unitMode; + } + + + public void setUnitMode(boolean isUnitMode) { + this.unitMode = isUnitMode; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/SendMessageRequestHeaderV2.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/SendMessageRequestHeaderV2.java new file mode 100644 index 000000000..ee1b71073 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/SendMessageRequestHeaderV2.java @@ -0,0 +1,188 @@ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.annotation.CFNullable; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * 为减少网络传输数量准备 + * + * @author shijia.wxr + */ +public class SendMessageRequestHeaderV2 implements CommandCustomHeader { + @CFNotNull + private String a;// producerGroup; + @CFNotNull + private String b;// topic; + @CFNotNull + private String c;// defaultTopic; + @CFNotNull + private Integer d;// defaultTopicQueueNums; + @CFNotNull + private Integer e;// queueId; + @CFNotNull + private Integer f;// sysFlag; + @CFNotNull + private Long g;// bornTimestamp; + @CFNotNull + private Integer h;// flag; + @CFNullable + private String i;// properties; + @CFNullable + private Integer j;// reconsumeTimes; + @CFNullable + private boolean k;// unitMode = false; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public static SendMessageRequestHeader createSendMessageRequestHeaderV1( + final SendMessageRequestHeaderV2 v2) { + SendMessageRequestHeader v1 = new SendMessageRequestHeader(); + v1.setProducerGroup(v2.a); + v1.setTopic(v2.b); + v1.setDefaultTopic(v2.c); + v1.setDefaultTopicQueueNums(v2.d); + v1.setQueueId(v2.e); + v1.setSysFlag(v2.f); + v1.setBornTimestamp(v2.g); + v1.setFlag(v2.h); + v1.setProperties(v2.i); + v1.setReconsumeTimes(v2.j); + v1.setUnitMode(v2.k); + return v1; + } + + + public static SendMessageRequestHeaderV2 createSendMessageRequestHeaderV2( + final SendMessageRequestHeader v1) { + SendMessageRequestHeaderV2 v2 = new SendMessageRequestHeaderV2(); + v2.a = v1.getProducerGroup(); + v2.b = v1.getTopic(); + v2.c = v1.getDefaultTopic(); + v2.d = v1.getDefaultTopicQueueNums(); + v2.e = v1.getQueueId(); + v2.f = v1.getSysFlag(); + v2.g = v1.getBornTimestamp(); + v2.h = v1.getFlag(); + v2.i = v1.getProperties(); + v2.j = v1.getReconsumeTimes(); + v2.k = v1.isUnitMode(); + return v2; + } + + + public String getA() { + return a; + } + + + public void setA(String a) { + this.a = a; + } + + + public String getB() { + return b; + } + + + public void setB(String b) { + this.b = b; + } + + + public String getC() { + return c; + } + + + public void setC(String c) { + this.c = c; + } + + + public Integer getD() { + return d; + } + + + public void setD(Integer d) { + this.d = d; + } + + + public Integer getE() { + return e; + } + + + public void setE(Integer e) { + this.e = e; + } + + + public Integer getF() { + return f; + } + + + public void setF(Integer f) { + this.f = f; + } + + + public Long getG() { + return g; + } + + + public void setG(Long g) { + this.g = g; + } + + + public Integer getH() { + return h; + } + + + public void setH(Integer h) { + this.h = h; + } + + + public String getI() { + return i; + } + + + public void setI(String i) { + this.i = i; + } + + + public Integer getJ() { + return j; + } + + + public void setJ(Integer j) { + this.j = j; + } + + + public boolean isK() { + return k; + } + + + public void setK(boolean k) { + this.k = k; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/SendMessageResponseHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/SendMessageResponseHeader.java new file mode 100644 index 000000000..a800e9553 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/SendMessageResponseHeader.java @@ -0,0 +1,56 @@ +/** + * $Id: SendMessageResponseHeader.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class SendMessageResponseHeader implements CommandCustomHeader { + @CFNotNull + private String msgId; + @CFNotNull + private Integer queueId; + @CFNotNull + private Long queueOffset; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public String getMsgId() { + return msgId; + } + + + public void setMsgId(String msgId) { + this.msgId = msgId; + } + + + public Integer getQueueId() { + return queueId; + } + + + public void setQueueId(Integer queueId) { + this.queueId = queueId; + } + + + public Long getQueueOffset() { + return queueOffset; + } + + + public void setQueueOffset(Long queueOffset) { + this.queueOffset = queueOffset; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/UnregisterClientRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/UnregisterClientRequestHeader.java new file mode 100644 index 000000000..d6e472688 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/UnregisterClientRequestHeader.java @@ -0,0 +1,60 @@ +/** + * + */ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.annotation.CFNullable; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class UnregisterClientRequestHeader implements CommandCustomHeader { + @CFNotNull + private String clientID; + + @CFNullable + private String producerGroup; + @CFNullable + private String consumerGroup; + + + public String getClientID() { + return clientID; + } + + + public void setClientID(String clientID) { + this.clientID = clientID; + } + + + public String getProducerGroup() { + return producerGroup; + } + + + public void setProducerGroup(String producerGroup) { + this.producerGroup = producerGroup; + } + + + public String getConsumerGroup() { + return consumerGroup; + } + + + public void setConsumerGroup(String consumerGroup) { + this.consumerGroup = consumerGroup; + } + + + @Override + public void checkFields() throws RemotingCommandException { + // TODO Auto-generated method stub + + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/UnregisterClientResponseHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/UnregisterClientResponseHeader.java new file mode 100644 index 000000000..fb25f0a82 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/UnregisterClientResponseHeader.java @@ -0,0 +1,26 @@ +/** + * + */ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class UnregisterClientResponseHeader implements CommandCustomHeader { + + /* + * (non-Javadoc) + * + * @see com.alibaba.rocketmq.remoting.CommandCustomHeader#checkFields() + */ + @Override + public void checkFields() throws RemotingCommandException { + // TODO Auto-generated method stub + + } + +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/UpdateConsumerOffsetRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/UpdateConsumerOffsetRequestHeader.java new file mode 100644 index 000000000..7e114f7cf --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/UpdateConsumerOffsetRequestHeader.java @@ -0,0 +1,68 @@ +/** + * $Id: UpdateConsumerOffsetRequestHeader.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class UpdateConsumerOffsetRequestHeader implements CommandCustomHeader { + @CFNotNull + private String consumerGroup; + @CFNotNull + private String topic; + @CFNotNull + private Integer queueId; + @CFNotNull + private Long commitOffset; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public String getConsumerGroup() { + return consumerGroup; + } + + + public void setConsumerGroup(String consumerGroup) { + this.consumerGroup = consumerGroup; + } + + + public String getTopic() { + return topic; + } + + + public void setTopic(String topic) { + this.topic = topic; + } + + + public Integer getQueueId() { + return queueId; + } + + + public void setQueueId(Integer queueId) { + this.queueId = queueId; + } + + + public Long getCommitOffset() { + return commitOffset; + } + + + public void setCommitOffset(Long commitOffset) { + this.commitOffset = commitOffset; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/UpdateConsumerOffsetResponseHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/UpdateConsumerOffsetResponseHeader.java new file mode 100644 index 000000000..e8791a69b --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/UpdateConsumerOffsetResponseHeader.java @@ -0,0 +1,18 @@ +/** + * $Id: UpdateConsumerOffsetResponseHeader.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class UpdateConsumerOffsetResponseHeader implements CommandCustomHeader { + @Override + public void checkFields() throws RemotingCommandException { + + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/ViewMessageRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/ViewMessageRequestHeader.java new file mode 100644 index 000000000..a01d5956a --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/ViewMessageRequestHeader.java @@ -0,0 +1,32 @@ +/** + * $Id: ViewMessageRequestHeader.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class ViewMessageRequestHeader implements CommandCustomHeader { + @CFNotNull + private Long offset; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public Long getOffset() { + return offset; + } + + + public void setOffset(Long offset) { + this.offset = offset; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/ViewMessageResponseHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/ViewMessageResponseHeader.java new file mode 100644 index 000000000..b48f650b1 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/ViewMessageResponseHeader.java @@ -0,0 +1,20 @@ +/** + * $Id: ViewMessageResponseHeader.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.header; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class ViewMessageResponseHeader implements CommandCustomHeader { + + @Override + public void checkFields() throws RemotingCommandException { + // TODO Auto-generated method stub + + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/filtersrv/RegisterFilterServerRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/filtersrv/RegisterFilterServerRequestHeader.java new file mode 100644 index 000000000..a2751d233 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/filtersrv/RegisterFilterServerRequestHeader.java @@ -0,0 +1,26 @@ +package com.alibaba.rocketmq.common.protocol.header.filtersrv; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +public class RegisterFilterServerRequestHeader implements CommandCustomHeader { + @CFNotNull + private String filterServerAddr; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public String getFilterServerAddr() { + return filterServerAddr; + } + + + public void setFilterServerAddr(String filterServerAddr) { + this.filterServerAddr = filterServerAddr; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/filtersrv/RegisterFilterServerResponseHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/filtersrv/RegisterFilterServerResponseHeader.java new file mode 100644 index 000000000..950303fa5 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/filtersrv/RegisterFilterServerResponseHeader.java @@ -0,0 +1,38 @@ +package com.alibaba.rocketmq.common.protocol.header.filtersrv; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +public class RegisterFilterServerResponseHeader implements CommandCustomHeader { + @CFNotNull + private String brokerName; + @CFNotNull + private long brokerId; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public long getBrokerId() { + return brokerId; + } + + + public void setBrokerId(long brokerId) { + this.brokerId = brokerId; + } + + + public String getBrokerName() { + return brokerName; + } + + + public void setBrokerName(String brokerName) { + this.brokerName = brokerName; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/filtersrv/RegisterMessageFilterClassRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/filtersrv/RegisterMessageFilterClassRequestHeader.java new file mode 100644 index 000000000..fc1eb5971 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/filtersrv/RegisterMessageFilterClassRequestHeader.java @@ -0,0 +1,62 @@ +package com.alibaba.rocketmq.common.protocol.header.filtersrv; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +public class RegisterMessageFilterClassRequestHeader implements CommandCustomHeader { + @CFNotNull + private String consumerGroup; + @CFNotNull + private String topic; + @CFNotNull + private String className; + @CFNotNull + private Integer classCRC; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public String getConsumerGroup() { + return consumerGroup; + } + + + public void setConsumerGroup(String consumerGroup) { + this.consumerGroup = consumerGroup; + } + + + public String getTopic() { + return topic; + } + + + public void setTopic(String topic) { + this.topic = topic; + } + + + public String getClassName() { + return className; + } + + + public void setClassName(String className) { + this.className = className; + } + + + public Integer getClassCRC() { + return classCRC; + } + + + public void setClassCRC(Integer classCRC) { + this.classCRC = classCRC; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/DeleteKVConfigRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/DeleteKVConfigRequestHeader.java new file mode 100644 index 000000000..d602ae9a9 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/DeleteKVConfigRequestHeader.java @@ -0,0 +1,42 @@ +package com.alibaba.rocketmq.common.protocol.header.namesrv; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + * @since 2013-7-1 + */ +public class DeleteKVConfigRequestHeader implements CommandCustomHeader { + @CFNotNull + private String namespace; + @CFNotNull + private String key; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public String getNamespace() { + return namespace; + } + + + public void setNamespace(String namespace) { + this.namespace = namespace; + } + + + public String getKey() { + return key; + } + + + public void setKey(String key) { + this.key = key; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/DeleteTopicInNamesrvRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/DeleteTopicInNamesrvRequestHeader.java new file mode 100644 index 000000000..0bd324a97 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/DeleteTopicInNamesrvRequestHeader.java @@ -0,0 +1,45 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common.protocol.header.namesrv; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + * @since 2013-8-11 + */ +public class DeleteTopicInNamesrvRequestHeader implements CommandCustomHeader { + @CFNotNull + private String topic; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public String getTopic() { + return topic; + } + + + public void setTopic(String topic) { + this.topic = topic; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/GetKVConfigRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/GetKVConfigRequestHeader.java new file mode 100644 index 000000000..93782fc43 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/GetKVConfigRequestHeader.java @@ -0,0 +1,42 @@ +package com.alibaba.rocketmq.common.protocol.header.namesrv; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + * @since 2013-7-1 + */ +public class GetKVConfigRequestHeader implements CommandCustomHeader { + @CFNotNull + private String namespace; + @CFNotNull + private String key; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public String getNamespace() { + return namespace; + } + + + public void setNamespace(String namespace) { + this.namespace = namespace; + } + + + public String getKey() { + return key; + } + + + public void setKey(String key) { + this.key = key; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/GetKVConfigResponseHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/GetKVConfigResponseHeader.java new file mode 100644 index 000000000..91d00e8f1 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/GetKVConfigResponseHeader.java @@ -0,0 +1,30 @@ +package com.alibaba.rocketmq.common.protocol.header.namesrv; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNullable; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + * @since 2013-7-1 + */ +public class GetKVConfigResponseHeader implements CommandCustomHeader { + @CFNullable + private String value; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public String getValue() { + return value; + } + + + public void setValue(String value) { + this.value = value; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/GetKVListByNamespaceRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/GetKVListByNamespaceRequestHeader.java new file mode 100644 index 000000000..f1fa6b57d --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/GetKVListByNamespaceRequestHeader.java @@ -0,0 +1,30 @@ +package com.alibaba.rocketmq.common.protocol.header.namesrv; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + * @since 2013-7-1 + */ +public class GetKVListByNamespaceRequestHeader implements CommandCustomHeader { + @CFNotNull + private String namespace; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public String getNamespace() { + return namespace; + } + + + public void setNamespace(String namespace) { + this.namespace = namespace; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/GetRouteInfoRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/GetRouteInfoRequestHeader.java new file mode 100644 index 000000000..b8ba6c937 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/GetRouteInfoRequestHeader.java @@ -0,0 +1,33 @@ +/** + * $Id: GetRouteInfoRequestHeader.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.header.namesrv; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class GetRouteInfoRequestHeader implements CommandCustomHeader { + @CFNotNull + private String topic; + + + @Override + public void checkFields() throws RemotingCommandException { + // TODO Auto-generated method stub + } + + + public String getTopic() { + return topic; + } + + + public void setTopic(String topic) { + this.topic = topic; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/GetRouteInfoResponseHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/GetRouteInfoResponseHeader.java new file mode 100644 index 000000000..06f46aa33 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/GetRouteInfoResponseHeader.java @@ -0,0 +1,20 @@ +/** + * $Id: GetRouteInfoResponseHeader.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.header.namesrv; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class GetRouteInfoResponseHeader implements CommandCustomHeader { + + @Override + public void checkFields() throws RemotingCommandException { + // TODO Auto-generated method stub + + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/PutKVConfigRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/PutKVConfigRequestHeader.java new file mode 100644 index 000000000..9f7edbb42 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/PutKVConfigRequestHeader.java @@ -0,0 +1,50 @@ +package com.alibaba.rocketmq.common.protocol.header.namesrv; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +public class PutKVConfigRequestHeader implements CommandCustomHeader { + @CFNotNull + private String namespace; + @CFNotNull + private String key; + @CFNotNull + private String value; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public String getNamespace() { + return namespace; + } + + + public void setNamespace(String namespace) { + this.namespace = namespace; + } + + + public String getKey() { + return key; + } + + + public void setKey(String key) { + this.key = key; + } + + + public String getValue() { + return value; + } + + + public void setValue(String value) { + this.value = value; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/RegisterBrokerRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/RegisterBrokerRequestHeader.java new file mode 100644 index 000000000..ee2dfdcbe --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/RegisterBrokerRequestHeader.java @@ -0,0 +1,80 @@ +/** + * $Id: RegisterBrokerRequestHeader.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.header.namesrv; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author lansheng.zj@taobao.com + */ +public class RegisterBrokerRequestHeader implements CommandCustomHeader { + @CFNotNull + private String brokerName; + @CFNotNull + private String brokerAddr; + @CFNotNull + private String clusterName; + @CFNotNull + private String haServerAddr; + @CFNotNull + private Long brokerId; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public String getBrokerName() { + return brokerName; + } + + + public void setBrokerName(String brokerName) { + this.brokerName = brokerName; + } + + + public String getBrokerAddr() { + return brokerAddr; + } + + + public void setBrokerAddr(String brokerAddr) { + this.brokerAddr = brokerAddr; + } + + + public String getClusterName() { + return clusterName; + } + + + public void setClusterName(String clusterName) { + this.clusterName = clusterName; + } + + + public String getHaServerAddr() { + return haServerAddr; + } + + + public void setHaServerAddr(String haServerAddr) { + this.haServerAddr = haServerAddr; + } + + + public Long getBrokerId() { + return brokerId; + } + + + public void setBrokerId(Long brokerId) { + this.brokerId = brokerId; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/RegisterBrokerResponseHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/RegisterBrokerResponseHeader.java new file mode 100644 index 000000000..d7199ff45 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/RegisterBrokerResponseHeader.java @@ -0,0 +1,42 @@ +package com.alibaba.rocketmq.common.protocol.header.namesrv; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNullable; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + * @since 2013-7-4 + */ +public class RegisterBrokerResponseHeader implements CommandCustomHeader { + @CFNullable + private String haServerAddr; + @CFNullable + private String masterAddr; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public String getHaServerAddr() { + return haServerAddr; + } + + + public void setHaServerAddr(String haServerAddr) { + this.haServerAddr = haServerAddr; + } + + + public String getMasterAddr() { + return masterAddr; + } + + + public void setMasterAddr(String masterAddr) { + this.masterAddr = masterAddr; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/RegisterOrderTopicRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/RegisterOrderTopicRequestHeader.java new file mode 100644 index 000000000..b86aac705 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/RegisterOrderTopicRequestHeader.java @@ -0,0 +1,45 @@ +/** + * $Id: RegisterOrderTopicRequestHeader.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.header.namesrv; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + */ +public class RegisterOrderTopicRequestHeader implements CommandCustomHeader { + @CFNotNull + private String topic; + @CFNotNull + private String orderTopicString; + + + @Override + public void checkFields() throws RemotingCommandException { + // TODO Auto-generated method stub + } + + + public String getTopic() { + return topic; + } + + + public void setTopic(String topic) { + this.topic = topic; + } + + + public String getOrderTopicString() { + return orderTopicString; + } + + + public void setOrderTopicString(String orderTopicString) { + this.orderTopicString = orderTopicString; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/UnRegisterBrokerRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/UnRegisterBrokerRequestHeader.java new file mode 100644 index 000000000..f6fb2e371 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/UnRegisterBrokerRequestHeader.java @@ -0,0 +1,68 @@ +/** + * $Id: UnRegisterBrokerRequestHeader.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.header.namesrv; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author lansheng.zj@taobao.com + */ +public class UnRegisterBrokerRequestHeader implements CommandCustomHeader { + @CFNotNull + private String brokerName; + @CFNotNull + private String brokerAddr; + @CFNotNull + private String clusterName; + @CFNotNull + private Long brokerId; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public String getBrokerName() { + return brokerName; + } + + + public void setBrokerName(String brokerName) { + this.brokerName = brokerName; + } + + + public String getBrokerAddr() { + return brokerAddr; + } + + + public void setBrokerAddr(String brokerAddr) { + this.brokerAddr = brokerAddr; + } + + + public String getClusterName() { + return clusterName; + } + + + public void setClusterName(String clusterName) { + this.clusterName = clusterName; + } + + + public Long getBrokerId() { + return brokerId; + } + + + public void setBrokerId(Long brokerId) { + this.brokerId = brokerId; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/WipeWritePermOfBrokerRequestHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/WipeWritePermOfBrokerRequestHeader.java new file mode 100644 index 000000000..25f853fcf --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/WipeWritePermOfBrokerRequestHeader.java @@ -0,0 +1,47 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common.protocol.header.namesrv; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + * @since 2013-8-5 + */ +public class WipeWritePermOfBrokerRequestHeader implements CommandCustomHeader { + @CFNotNull + private String brokerName; + + + @Override + public void checkFields() throws RemotingCommandException { + // TODO Auto-generated method stub + + } + + + public String getBrokerName() { + return brokerName; + } + + + public void setBrokerName(String brokerName) { + this.brokerName = brokerName; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/WipeWritePermOfBrokerResponseHeader.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/WipeWritePermOfBrokerResponseHeader.java new file mode 100644 index 000000000..7cfe2cffc --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/header/namesrv/WipeWritePermOfBrokerResponseHeader.java @@ -0,0 +1,45 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common.protocol.header.namesrv; + +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * @author shijia.wxr + * @since 2013-8-5 + */ +public class WipeWritePermOfBrokerResponseHeader implements CommandCustomHeader { + @CFNotNull + private Integer wipeTopicCount; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public Integer getWipeTopicCount() { + return wipeTopicCount; + } + + + public void setWipeTopicCount(Integer wipeTopicCount) { + this.wipeTopicCount = wipeTopicCount; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/heartbeat/ConsumeType.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/heartbeat/ConsumeType.java new file mode 100644 index 000000000..bd226c6fb --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/heartbeat/ConsumeType.java @@ -0,0 +1,20 @@ +/** + * $Id: ConsumeType.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.heartbeat; + +/** + * 消费类型 + * + * @author shijia.wxr + */ +public enum ConsumeType { + /** + * 主动方式消费 + */ + CONSUME_ACTIVELY, + /** + * 被动方式消费 + */ + CONSUME_PASSIVELY, +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/heartbeat/ConsumerData.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/heartbeat/ConsumerData.java new file mode 100644 index 000000000..972f6ac71 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/heartbeat/ConsumerData.java @@ -0,0 +1,90 @@ +/** + * $Id: ConsumerData.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.heartbeat; + +import java.util.HashSet; +import java.util.Set; + +import com.alibaba.rocketmq.common.consumer.ConsumeFromWhere; + + +/** + * @author shijia.wxr + */ +public class ConsumerData { + private String groupName; + private ConsumeType consumeType; + private MessageModel messageModel; + private ConsumeFromWhere consumeFromWhere; + private Set subscriptionDataSet = new HashSet(); + private boolean unitMode; + + + public String getGroupName() { + return groupName; + } + + + public void setGroupName(String groupName) { + this.groupName = groupName; + } + + + public ConsumeType getConsumeType() { + return consumeType; + } + + + public void setConsumeType(ConsumeType consumeType) { + this.consumeType = consumeType; + } + + + public MessageModel getMessageModel() { + return messageModel; + } + + + public void setMessageModel(MessageModel messageModel) { + this.messageModel = messageModel; + } + + + public ConsumeFromWhere getConsumeFromWhere() { + return consumeFromWhere; + } + + + public void setConsumeFromWhere(ConsumeFromWhere consumeFromWhere) { + this.consumeFromWhere = consumeFromWhere; + } + + + public Set getSubscriptionDataSet() { + return subscriptionDataSet; + } + + + public void setSubscriptionDataSet(Set subscriptionDataSet) { + this.subscriptionDataSet = subscriptionDataSet; + } + + + public boolean isUnitMode() { + return unitMode; + } + + + public void setUnitMode(boolean isUnitMode) { + this.unitMode = isUnitMode; + } + + + @Override + public String toString() { + return "ConsumerData [groupName=" + groupName + ", consumeType=" + consumeType + ", messageModel=" + + messageModel + ", consumeFromWhere=" + consumeFromWhere + ", unitMode=" + unitMode + + ", subscriptionDataSet=" + subscriptionDataSet + "]"; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/heartbeat/HeartbeatData.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/heartbeat/HeartbeatData.java new file mode 100644 index 000000000..df496e6f0 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/heartbeat/HeartbeatData.java @@ -0,0 +1,56 @@ +/** + * $Id: HeartbeatData.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.heartbeat; + +import java.util.HashSet; +import java.util.Set; + +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; + + +/** + * @author shijia.wxr + */ +public class HeartbeatData extends RemotingSerializable { + private String clientID; + private Set producerDataSet = new HashSet(); + private Set consumerDataSet = new HashSet(); + + + public String getClientID() { + return clientID; + } + + + public void setClientID(String clientID) { + this.clientID = clientID; + } + + + public Set getProducerDataSet() { + return producerDataSet; + } + + + public void setProducerDataSet(Set producerDataSet) { + this.producerDataSet = producerDataSet; + } + + + public Set getConsumerDataSet() { + return consumerDataSet; + } + + + public void setConsumerDataSet(Set consumerDataSet) { + this.consumerDataSet = consumerDataSet; + } + + + @Override + public String toString() { + return "HeartbeatData [clientID=" + clientID + ", producerDataSet=" + producerDataSet + + ", consumerDataSet=" + consumerDataSet + "]"; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/heartbeat/MessageModel.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/heartbeat/MessageModel.java new file mode 100644 index 000000000..473e2a96a --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/heartbeat/MessageModel.java @@ -0,0 +1,24 @@ +/** + * $Id: MessageModel.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.heartbeat; + +/** + * 消息模型 + * + * @author shijia.wxr + */ +public enum MessageModel { + /** + * 广播模型 + */ + BROADCASTING, + /** + * 集群模型 + */ + CLUSTERING, + // /** + // * 未知，如果是主动消费，很难确定应用的消息模型 + // */ + // UNKNOWNS, +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/heartbeat/ProducerData.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/heartbeat/ProducerData.java new file mode 100644 index 000000000..d20ee2ce3 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/heartbeat/ProducerData.java @@ -0,0 +1,27 @@ +/** + * $Id: ProducerData.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.heartbeat; + +/** + * @author shijia.wxr + */ +public class ProducerData { + private String groupName; + + + public String getGroupName() { + return groupName; + } + + + public void setGroupName(String groupName) { + this.groupName = groupName; + } + + + @Override + public String toString() { + return "ProducerData [groupName=" + groupName + "]"; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/heartbeat/SubscriptionData.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/heartbeat/SubscriptionData.java new file mode 100644 index 000000000..6ff72d3a0 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/heartbeat/SubscriptionData.java @@ -0,0 +1,163 @@ +/** + * $Id: SubscriptionData.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.heartbeat; + +import java.util.HashSet; +import java.util.Set; + + +/** + * @author shijia.wxr + */ +public class SubscriptionData implements Comparable { + public final static String SUB_ALL = "*"; + private boolean classFilterMode = false; + private String topic; + private String subString; + private Set tagsSet = new HashSet(); + private Set codeSet = new HashSet(); + private long subVersion = System.currentTimeMillis(); + + + public SubscriptionData() { + + } + + + public SubscriptionData(String topic, String subString) { + super(); + this.topic = topic; + this.subString = subString; + } + + + public String getTopic() { + return topic; + } + + + public void setTopic(String topic) { + this.topic = topic; + } + + + public String getSubString() { + return subString; + } + + + public void setSubString(String subString) { + this.subString = subString; + } + + + public Set getTagsSet() { + return tagsSet; + } + + + public void setTagsSet(Set tagsSet) { + this.tagsSet = tagsSet; + } + + + public long getSubVersion() { + return subVersion; + } + + + public void setSubVersion(long subVersion) { + this.subVersion = subVersion; + } + + + public Set getCodeSet() { + return codeSet; + } + + + public void setCodeSet(Set codeSet) { + this.codeSet = codeSet; + } + + + public boolean isClassFilterMode() { + return classFilterMode; + } + + + public void setClassFilterMode(boolean classFilterMode) { + this.classFilterMode = classFilterMode; + } + + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + (classFilterMode ? 1231 : 1237); + result = prime * result + ((codeSet == null) ? 0 : codeSet.hashCode()); + result = prime * result + ((subString == null) ? 0 : subString.hashCode()); + result = prime * result + ((tagsSet == null) ? 0 : tagsSet.hashCode()); + result = prime * result + ((topic == null) ? 0 : topic.hashCode()); + return result; + } + + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + SubscriptionData other = (SubscriptionData) obj; + if (classFilterMode != other.classFilterMode) + return false; + if (codeSet == null) { + if (other.codeSet != null) + return false; + } + else if (!codeSet.equals(other.codeSet)) + return false; + if (subString == null) { + if (other.subString != null) + return false; + } + else if (!subString.equals(other.subString)) + return false; + if (subVersion != other.subVersion) + return false; + if (tagsSet == null) { + if (other.tagsSet != null) + return false; + } + else if (!tagsSet.equals(other.tagsSet)) + return false; + if (topic == null) { + if (other.topic != null) + return false; + } + else if (!topic.equals(other.topic)) + return false; + return true; + } + + + @Override + public String toString() { + return "SubscriptionData [classFilterMode=" + classFilterMode + ", topic=" + topic + ", subString=" + + subString + ", tagsSet=" + tagsSet + ", codeSet=" + codeSet + ", subVersion=" + subVersion + + "]"; + } + + + @Override + public int compareTo(SubscriptionData other) { + String thisValue = this.topic + "@" + this.subString; + String otherValue = other.topic + "@" + other.subString; + return thisValue.compareTo(otherValue); + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/route/BrokerData.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/route/BrokerData.java new file mode 100644 index 000000000..09ae06888 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/route/BrokerData.java @@ -0,0 +1,100 @@ +/** + * $Id: BrokerData.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.route; + +import java.util.HashMap; + +import com.alibaba.rocketmq.common.MixAll; + + +/** + * @author shijia.wxr + * @since 2013-7-2 + */ +public class BrokerData implements Comparable { + private String brokerName; + private HashMap brokerAddrs; + + + /** + * 优先获取Master，如果没有Master尝试找Slave + */ + public String selectBrokerAddr() { + String value = this.brokerAddrs.get(MixAll.MASTER_ID); + if (null == value) { + for (Long key : this.brokerAddrs.keySet()) { + return this.brokerAddrs.get(key); + } + } + + return value; + } + + + public String getBrokerName() { + return brokerName; + } + + + public void setBrokerName(String brokerName) { + this.brokerName = brokerName; + } + + + public HashMap getBrokerAddrs() { + return brokerAddrs; + } + + + public void setBrokerAddrs(HashMap brokerAddrs) { + this.brokerAddrs = brokerAddrs; + } + + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((brokerAddrs == null) ? 0 : brokerAddrs.hashCode()); + result = prime * result + ((brokerName == null) ? 0 : brokerName.hashCode()); + return result; + } + + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + BrokerData other = (BrokerData) obj; + if (brokerAddrs == null) { + if (other.brokerAddrs != null) + return false; + } + else if (!brokerAddrs.equals(other.brokerAddrs)) + return false; + if (brokerName == null) { + if (other.brokerName != null) + return false; + } + else if (!brokerName.equals(other.brokerName)) + return false; + return true; + } + + + @Override + public String toString() { + return "BrokerData [brokerName=" + brokerName + ", brokerAddrs=" + brokerAddrs + "]"; + } + + + @Override + public int compareTo(BrokerData o) { + return this.brokerName.compareTo(o.getBrokerName()); + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/route/QueueData.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/route/QueueData.java new file mode 100644 index 000000000..34a80bd79 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/route/QueueData.java @@ -0,0 +1,116 @@ +/** + * $Id: QueueData.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.route; + +public class QueueData implements Comparable { + private String brokerName; + private int readQueueNums; + private int writeQueueNums; + private int perm; + private int topicSynFlag; + + + public String getBrokerName() { + return brokerName; + } + + + public void setBrokerName(String brokerName) { + this.brokerName = brokerName; + } + + + public int getReadQueueNums() { + return readQueueNums; + } + + + public void setReadQueueNums(int readQueueNums) { + this.readQueueNums = readQueueNums; + } + + + public int getWriteQueueNums() { + return writeQueueNums; + } + + + public void setWriteQueueNums(int writeQueueNums) { + this.writeQueueNums = writeQueueNums; + } + + + public int getPerm() { + return perm; + } + + + public void setPerm(int perm) { + this.perm = perm; + } + + + public int getTopicSynFlag() { + return topicSynFlag; + } + + + public void setTopicSynFlag(int topicSynFlag) { + this.topicSynFlag = topicSynFlag; + } + + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((brokerName == null) ? 0 : brokerName.hashCode()); + result = prime * result + perm; + result = prime * result + readQueueNums; + result = prime * result + writeQueueNums; + result = prime * result + topicSynFlag; + return result; + } + + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + QueueData other = (QueueData) obj; + if (brokerName == null) { + if (other.brokerName != null) + return false; + } + else if (!brokerName.equals(other.brokerName)) + return false; + if (perm != other.perm) + return false; + if (readQueueNums != other.readQueueNums) + return false; + if (writeQueueNums != other.writeQueueNums) + return false; + if (topicSynFlag != other.topicSynFlag) + return false; + return true; + } + + + @Override + public String toString() { + return "QueueData [brokerName=" + brokerName + ", readQueueNums=" + readQueueNums + + ", writeQueueNums=" + writeQueueNums + ", perm=" + perm + ", topicSynFlag=" + topicSynFlag + + "]"; + } + + + @Override + public int compareTo(QueueData o) { + return this.brokerName.compareTo(o.getBrokerName()); + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/route/TopicRouteData.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/route/TopicRouteData.java new file mode 100644 index 000000000..c4803f25c --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/route/TopicRouteData.java @@ -0,0 +1,142 @@ +/** + * $Id: TopicRouteData.java 1835 2013-05-16 02:00:50Z shijia.wxr $ + */ +package com.alibaba.rocketmq.common.protocol.route; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; + + +/** + * Topic路由数据，从Name Server获取 + * + * @author shijia.wxr + */ +public class TopicRouteData extends RemotingSerializable { + private String orderTopicConf; + private List queueDatas; + private List brokerDatas; + private HashMap/* Filter Server */> filterServerTable; + + + public TopicRouteData cloneTopicRouteData() { + TopicRouteData topicRouteData = new TopicRouteData(); + topicRouteData.setQueueDatas(new ArrayList()); + topicRouteData.setBrokerDatas(new ArrayList()); + topicRouteData.setFilterServerTable(new HashMap>()); + topicRouteData.setOrderTopicConf(this.orderTopicConf); + + if (this.queueDatas != null) { + topicRouteData.getQueueDatas().addAll(this.queueDatas); + } + + if (this.brokerDatas != null) { + topicRouteData.getBrokerDatas().addAll(this.brokerDatas); + } + + if (this.filterServerTable != null) { + topicRouteData.getFilterServerTable().putAll(this.filterServerTable); + } + + return topicRouteData; + } + + + public List getQueueDatas() { + return queueDatas; + } + + + public void setQueueDatas(List queueDatas) { + this.queueDatas = queueDatas; + } + + + public List getBrokerDatas() { + return brokerDatas; + } + + + public void setBrokerDatas(List brokerDatas) { + this.brokerDatas = brokerDatas; + } + + + public String getOrderTopicConf() { + return orderTopicConf; + } + + + public void setOrderTopicConf(String orderTopicConf) { + this.orderTopicConf = orderTopicConf; + } + + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((brokerDatas == null) ? 0 : brokerDatas.hashCode()); + result = prime * result + ((orderTopicConf == null) ? 0 : orderTopicConf.hashCode()); + result = prime * result + ((queueDatas == null) ? 0 : queueDatas.hashCode()); + result = prime * result + ((filterServerTable == null) ? 0 : filterServerTable.hashCode()); + return result; + } + + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + TopicRouteData other = (TopicRouteData) obj; + if (brokerDatas == null) { + if (other.brokerDatas != null) + return false; + } + else if (!brokerDatas.equals(other.brokerDatas)) + return false; + if (orderTopicConf == null) { + if (other.orderTopicConf != null) + return false; + } + else if (!orderTopicConf.equals(other.orderTopicConf)) + return false; + if (queueDatas == null) { + if (other.queueDatas != null) + return false; + } + else if (!queueDatas.equals(other.queueDatas)) + return false; + if (filterServerTable == null) { + if (other.filterServerTable != null) + return false; + } + else if (!filterServerTable.equals(other.filterServerTable)) + return false; + return true; + } + + + public HashMap> getFilterServerTable() { + return filterServerTable; + } + + + public void setFilterServerTable(HashMap> filterServerTable) { + this.filterServerTable = filterServerTable; + } + + + @Override + public String toString() { + return "TopicRouteData [orderTopicConf=" + orderTopicConf + ", queueDatas=" + queueDatas + + ", brokerDatas=" + brokerDatas + ", filterServerTable=" + filterServerTable + "]"; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/topic/OffsetMovedEvent.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/topic/OffsetMovedEvent.java new file mode 100644 index 000000000..7ec321e0f --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/protocol/topic/OffsetMovedEvent.java @@ -0,0 +1,65 @@ +package com.alibaba.rocketmq.common.protocol.topic; + +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; + + +public class OffsetMovedEvent extends RemotingSerializable { + private String consumerGroup; + private MessageQueue messageQueue; + /** + * 客户端请求的Offset + */ + private long offsetRequest; + /** + * Broker要求从这个新的Offset开始消费 + */ + private long offsetNew; + + + public String getConsumerGroup() { + return consumerGroup; + } + + + public void setConsumerGroup(String consumerGroup) { + this.consumerGroup = consumerGroup; + } + + + public MessageQueue getMessageQueue() { + return messageQueue; + } + + + public void setMessageQueue(MessageQueue messageQueue) { + this.messageQueue = messageQueue; + } + + + public long getOffsetRequest() { + return offsetRequest; + } + + + public void setOffsetRequest(long offsetRequest) { + this.offsetRequest = offsetRequest; + } + + + public long getOffsetNew() { + return offsetNew; + } + + + public void setOffsetNew(long offsetNew) { + this.offsetNew = offsetNew; + } + + + @Override + public String toString() { + return "OffsetMovedEvent [consumerGroup=" + consumerGroup + ", messageQueue=" + messageQueue + + ", offsetRequest=" + offsetRequest + ", offsetNew=" + offsetNew + "]"; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/running/RunningStats.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/running/RunningStats.java new file mode 100644 index 000000000..b4e6737cb --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/running/RunningStats.java @@ -0,0 +1,24 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common.running; + +public enum RunningStats { + commitLogMaxOffset, + commitLogMinOffset, + commitLogDiskRatio, + consumeQueueDiskRatio, + scheduleMessageOffset, +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/stats/StatsItem.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/stats/StatsItem.java new file mode 100644 index 000000000..ffb3661b9 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/stats/StatsItem.java @@ -0,0 +1,277 @@ +package com.alibaba.rocketmq.common.stats; + +import java.util.LinkedList; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +import org.slf4j.Logger; + +import com.alibaba.rocketmq.common.UtilAll; + + +public class StatsItem { + // 具体的统计值 + private final AtomicLong value = new AtomicLong(0); + // 统计次数 + private final AtomicLong times = new AtomicLong(0); + // 最近一分钟内的镜像，数量6，10秒钟采样一次 + private final LinkedList csListMinute = new LinkedList(); + + // 最近一小时内的镜像，数量6，10分钟采样一次 + private final LinkedList csListHour = new LinkedList(); + + // 最近一天内的镜像，数量24，1小时采样一次 + private final LinkedList csListDay = new LinkedList(); + + private final String statsName; + private final String statsKey; + private final ScheduledExecutorService scheduledExecutorService; + private final Logger log; + + + private static StatsSnapshot computeStatsData(final LinkedList csList) { + StatsSnapshot statsSnapshot = new StatsSnapshot(); + synchronized (csList) { + double tps = 0; + double avgpt = 0; + long sum = 0; + if (!csList.isEmpty()) { + CallSnapshot first = csList.getFirst(); + CallSnapshot last = csList.getLast(); + sum = last.getValue() - first.getValue(); + tps = (sum * 1000.0d) / (last.getTimestamp() - first.getTimestamp()); + + long timesDiff = last.getTimes() - first.getTimes(); + if (timesDiff > 0) { + avgpt = (sum * 1.0d) / (timesDiff); + } + } + + statsSnapshot.setSum(sum); + statsSnapshot.setTps(tps); + statsSnapshot.setAvgpt(avgpt); + } + + return statsSnapshot; + } + + + public StatsSnapshot getStatsDataInMinute() { + return computeStatsData(this.csListMinute); + } + + + public StatsSnapshot getStatsDataInHour() { + return computeStatsData(this.csListHour); + } + + + public StatsSnapshot getStatsDataInDay() { + return computeStatsData(this.csListDay); + } + + + public StatsItem(String statsName, String statsKey, ScheduledExecutorService scheduledExecutorService, + Logger log) { + this.statsName = statsName; + this.statsKey = statsKey; + this.scheduledExecutorService = scheduledExecutorService; + this.log = log; + } + + + public void init() { + // 每隔10s执行一次 + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + try { + samplingInSeconds(); + } + catch (Throwable e) { + } + } + }, 0, 10, TimeUnit.SECONDS); + + // 每隔10分钟执行一次 + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + try { + samplingInMinutes(); + } + catch (Throwable e) { + } + } + }, 0, 10, TimeUnit.MINUTES); + + // 每隔1小时执行一次 + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + try { + samplingInHour(); + } + catch (Throwable e) { + } + } + }, 0, 1, TimeUnit.HOURS); + + // 分钟整点执行 + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + try { + printAtMinutes(); + } + catch (Throwable e) { + } + } + }, Math.abs(UtilAll.computNextMinutesTimeMillis() - System.currentTimeMillis()), // + 1000 * 60, TimeUnit.MILLISECONDS); + + // 小时整点执行 + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + try { + printAtHour(); + } + catch (Throwable e) { + } + } + }, Math.abs(UtilAll.computNextHourTimeMillis() - System.currentTimeMillis()), // + 1000 * 60 * 60, TimeUnit.MILLISECONDS); + + // 每天0点执行 + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + try { + printAtDay(); + } + catch (Throwable e) { + } + } + }, Math.abs(UtilAll.computNextMorningTimeMillis() - System.currentTimeMillis()) - 2000, // + 1000 * 60 * 60 * 24, TimeUnit.MILLISECONDS); + } + + + public void printAtMinutes() { + StatsSnapshot ss = computeStatsData(this.csListMinute); + log.info(String.format("[%s] [%s] Stats In One Minute, SUM: %d TPS: %.2f AVGPT: %.2f", // + this.statsName,// + this.statsKey,// + ss.getSum(),// + ss.getTps(),// + ss.getAvgpt())); + } + + + public void printAtHour() { + StatsSnapshot ss = computeStatsData(this.csListHour); + log.info(String.format("[%s] [%s] Stats In One Hour, SUM: %d TPS: %.2f AVGPT: %.2f", // + this.statsName,// + this.statsKey,// + ss.getSum(),// + ss.getTps(),// + ss.getAvgpt())); + } + + + public void printAtDay() { + StatsSnapshot ss = computeStatsData(this.csListDay); + log.info(String.format("[%s] [%s] Stats In One Day, SUM: %d TPS: %.2f AVGPT: %.2f", // + this.statsName,// + this.statsKey,// + ss.getSum(),// + ss.getTps(),// + ss.getAvgpt())); + } + + + public void samplingInSeconds() { + synchronized (this.csListMinute) { + this.csListMinute.add(new CallSnapshot(System.currentTimeMillis(), this.times.get(), this.value + .get())); + if (this.csListMinute.size() > 7) { + this.csListMinute.removeFirst(); + } + } + } + + + public void samplingInMinutes() { + synchronized (this.csListHour) { + this.csListHour.add(new CallSnapshot(System.currentTimeMillis(), this.times.get(), this.value + .get())); + if (this.csListHour.size() > 7) { + this.csListHour.removeFirst(); + } + } + } + + + public void samplingInHour() { + synchronized (this.csListDay) { + this.csListDay.add(new CallSnapshot(System.currentTimeMillis(), this.times.get(), this.value + .get())); + if (this.csListDay.size() > 25) { + this.csListDay.removeFirst(); + } + } + } + + + public AtomicLong getValue() { + return value; + } + + + public String getStatsKey() { + return statsKey; + } + + + public String getStatsName() { + return statsName; + } + + + public AtomicLong getTimes() { + return times; + } +} + + +class CallSnapshot { + private final long timestamp; + private final long times; + + private final long value; + + + public CallSnapshot(long timestamp, long times, long value) { + super(); + this.timestamp = timestamp; + this.times = times; + this.value = value; + } + + + public long getTimestamp() { + return timestamp; + } + + + public long getTimes() { + return times; + } + + + public long getValue() { + return value; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/stats/StatsItemSet.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/stats/StatsItemSet.java new file mode 100644 index 000000000..c53a1dacf --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/stats/StatsItemSet.java @@ -0,0 +1,210 @@ +package com.alibaba.rocketmq.common.stats; + +import com.alibaba.rocketmq.common.UtilAll; +import org.slf4j.Logger; + +import java.util.Iterator; +import java.util.Map.Entry; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + + +public class StatsItemSet { + private final ConcurrentHashMap statsItemTable = + new ConcurrentHashMap(128); + + private final String statsName; + private final ScheduledExecutorService scheduledExecutorService; + private final Logger log; + + + public StatsItemSet(String statsName, ScheduledExecutorService scheduledExecutorService, Logger log) { + this.statsName = statsName; + this.scheduledExecutorService = scheduledExecutorService; + this.log = log; + this.init(); + } + + + private StatsItem getAndCreateStatsItem(final String statsKey) { + StatsItem statsItem = this.statsItemTable.get(statsKey); + if (null == statsItem) { + statsItem = new StatsItem(this.statsName, statsKey, this.scheduledExecutorService, this.log); + StatsItem prev = this.statsItemTable.put(statsKey, statsItem); + // 说明是第一次插入 + if (null == prev) { + // 内部不需要定时，外部统一定时 + // statsItem.init(); + } + } + + return statsItem; + } + + + public void addValue(final String statsKey, final int incValue, final int incTimes) { + StatsItem statsItem = this.getAndCreateStatsItem(statsKey); + statsItem.getValue().addAndGet(incValue); + statsItem.getTimes().addAndGet(incTimes); + } + + + public StatsSnapshot getStatsDataInMinute(final String statsKey) { + StatsItem statsItem = this.statsItemTable.get(statsKey); + if (null != statsItem) { + return statsItem.getStatsDataInMinute(); + } + return new StatsSnapshot(); + } + + + public StatsSnapshot getStatsDataInHour(final String statsKey) { + StatsItem statsItem = this.statsItemTable.get(statsKey); + if (null != statsItem) { + return statsItem.getStatsDataInHour(); + } + return new StatsSnapshot(); + } + + + public StatsSnapshot getStatsDataInDay(final String statsKey) { + StatsItem statsItem = this.statsItemTable.get(statsKey); + if (null != statsItem) { + return statsItem.getStatsDataInDay(); + } + return new StatsSnapshot(); + } + + + public void init() { + // 每隔10s执行一次 + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + try { + samplingInSeconds(); + } + catch (Throwable e) { + } + } + }, 0, 10, TimeUnit.SECONDS); + + // 每隔10分钟执行一次 + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + try { + samplingInMinutes(); + } + catch (Throwable e) { + } + } + }, 0, 10, TimeUnit.MINUTES); + + // 每隔1小时执行一次 + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + try { + samplingInHour(); + } + catch (Throwable e) { + } + } + }, 0, 1, TimeUnit.HOURS); + + // 分钟整点执行 + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + try { + printAtMinutes(); + } + catch (Throwable e) { + } + } + }, Math.abs(UtilAll.computNextMinutesTimeMillis() - System.currentTimeMillis()), // + 1000 * 60, TimeUnit.MILLISECONDS); + + // 小时整点执行 + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + try { + printAtHour(); + } + catch (Throwable e) { + } + } + }, Math.abs(UtilAll.computNextHourTimeMillis() - System.currentTimeMillis()), // + 1000 * 60 * 60, TimeUnit.MILLISECONDS); + + // 每天0点执行 + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + try { + printAtDay(); + } + catch (Throwable e) { + } + } + }, Math.abs(UtilAll.computNextMorningTimeMillis() - System.currentTimeMillis()), // + 1000 * 60 * 60 * 24, TimeUnit.MILLISECONDS); + } + + + private void printAtMinutes() { + Iterator> it = this.statsItemTable.entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + next.getValue().printAtMinutes(); + } + } + + + private void printAtHour() { + Iterator> it = this.statsItemTable.entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + next.getValue().printAtHour(); + } + } + + + private void printAtDay() { + Iterator> it = this.statsItemTable.entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + next.getValue().printAtDay(); + } + } + + + private void samplingInSeconds() { + Iterator> it = this.statsItemTable.entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + next.getValue().samplingInSeconds(); + } + } + + + private void samplingInMinutes() { + Iterator> it = this.statsItemTable.entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + next.getValue().samplingInMinutes(); + } + } + + + private void samplingInHour() { + Iterator> it = this.statsItemTable.entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + next.getValue().samplingInHour(); + } + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/stats/StatsSnapshot.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/stats/StatsSnapshot.java new file mode 100644 index 000000000..839ca5397 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/stats/StatsSnapshot.java @@ -0,0 +1,37 @@ +package com.alibaba.rocketmq.common.stats; + +public class StatsSnapshot { + private long sum; + private double tps; + private double avgpt; + + + public long getSum() { + return sum; + } + + + public void setSum(long sum) { + this.sum = sum; + } + + + public double getTps() { + return tps; + } + + + public void setTps(double tps) { + this.tps = tps; + } + + + public double getAvgpt() { + return avgpt; + } + + + public void setAvgpt(double avgpt) { + this.avgpt = avgpt; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/subscription/SubscriptionGroupConfig.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/subscription/SubscriptionGroupConfig.java new file mode 100644 index 000000000..b4363012a --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/subscription/SubscriptionGroupConfig.java @@ -0,0 +1,167 @@ +package com.alibaba.rocketmq.common.subscription; + +import com.alibaba.rocketmq.common.MixAll; + + +/** + * @author shijia.wxr + * @since 2013-6-18 + */ +public class SubscriptionGroupConfig { + // 订阅组名 + private String groupName; + // 消费功能是否开启 + private boolean consumeEnable = true; + // 是否允许从队列最小位置开始消费，线上默认会设置为false + private boolean consumeFromMinEnable = true; + // 是否允许广播方式消费 + private boolean consumeBroadcastEnable = true; + // 消费失败的消息放到一个重试队列，每个订阅组配置几个重试队列 + private int retryQueueNums = 1; + // 重试消费最大次数，超过则投递到死信队列，不再投递，并报警 + private int retryMaxTimes = 16; + // 从哪个Broker开始消费 + private long brokerId = MixAll.MASTER_ID; + // 发现消息堆积后，将Consumer的消费请求重定向到另外一台Slave机器 + private long whichBrokerWhenConsumeSlowly = 1; + + + public String getGroupName() { + return groupName; + } + + + public void setGroupName(String groupName) { + this.groupName = groupName; + } + + + public boolean isConsumeEnable() { + return consumeEnable; + } + + + public void setConsumeEnable(boolean consumeEnable) { + this.consumeEnable = consumeEnable; + } + + + public boolean isConsumeFromMinEnable() { + return consumeFromMinEnable; + } + + + public void setConsumeFromMinEnable(boolean consumeFromMinEnable) { + this.consumeFromMinEnable = consumeFromMinEnable; + } + + + public boolean isConsumeBroadcastEnable() { + return consumeBroadcastEnable; + } + + + public void setConsumeBroadcastEnable(boolean consumeBroadcastEnable) { + this.consumeBroadcastEnable = consumeBroadcastEnable; + } + + + public int getRetryQueueNums() { + return retryQueueNums; + } + + + public void setRetryQueueNums(int retryQueueNums) { + this.retryQueueNums = retryQueueNums; + } + + + public int getRetryMaxTimes() { + return retryMaxTimes; + } + + + public void setRetryMaxTimes(int retryMaxTimes) { + this.retryMaxTimes = retryMaxTimes; + } + + + public long getBrokerId() { + return brokerId; + } + + + public void setBrokerId(long brokerId) { + this.brokerId = brokerId; + } + + + public long getWhichBrokerWhenConsumeSlowly() { + return whichBrokerWhenConsumeSlowly; + } + + + public void setWhichBrokerWhenConsumeSlowly(long whichBrokerWhenConsumeSlowly) { + this.whichBrokerWhenConsumeSlowly = whichBrokerWhenConsumeSlowly; + } + + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + (int) (brokerId ^ (brokerId >>> 32)); + result = prime * result + (consumeBroadcastEnable ? 1231 : 1237); + result = prime * result + (consumeEnable ? 1231 : 1237); + result = prime * result + (consumeFromMinEnable ? 1231 : 1237); + result = prime * result + ((groupName == null) ? 0 : groupName.hashCode()); + result = prime * result + retryMaxTimes; + result = prime * result + retryQueueNums; + result = + prime * result + (int) (whichBrokerWhenConsumeSlowly ^ (whichBrokerWhenConsumeSlowly >>> 32)); + return result; + } + + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + SubscriptionGroupConfig other = (SubscriptionGroupConfig) obj; + if (brokerId != other.brokerId) + return false; + if (consumeBroadcastEnable != other.consumeBroadcastEnable) + return false; + if (consumeEnable != other.consumeEnable) + return false; + if (consumeFromMinEnable != other.consumeFromMinEnable) + return false; + if (groupName == null) { + if (other.groupName != null) + return false; + } + else if (!groupName.equals(other.groupName)) + return false; + if (retryMaxTimes != other.retryMaxTimes) + return false; + if (retryQueueNums != other.retryQueueNums) + return false; + if (whichBrokerWhenConsumeSlowly != other.whichBrokerWhenConsumeSlowly) + return false; + return true; + } + + + @Override + public String toString() { + return "SubscriptionGroupConfig [groupName=" + groupName + ", consumeEnable=" + consumeEnable + + ", consumeFromMinEnable=" + consumeFromMinEnable + ", consumeBroadcastEnable=" + + consumeBroadcastEnable + ", retryQueueNums=" + retryQueueNums + ", retryMaxTimes=" + + retryMaxTimes + ", brokerId=" + brokerId + ", whichBrokerWhenConsumeSlowly=" + + whichBrokerWhenConsumeSlowly + "]"; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/sysflag/MessageSysFlag.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/sysflag/MessageSysFlag.java new file mode 100644 index 000000000..712783533 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/sysflag/MessageSysFlag.java @@ -0,0 +1,51 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common.sysflag; + +/** + * @author shijia.wxr + */ +public class MessageSysFlag { + /** + * SysFlag + */ + public final static int CompressedFlag = (0x1 << 0); + public final static int MultiTagsFlag = (0x1 << 1); + + /** + * 7 6 5 4 3 2 1 0
+ * SysFlag 事务相关，从左属，2与3 + */ + public final static int TransactionNotType = (0x0 << 2); + public final static int TransactionPreparedType = (0x1 << 2); + public final static int TransactionCommitType = (0x2 << 2); + public final static int TransactionRollbackType = (0x3 << 2); + + + public static int getTransactionValue(final int flag) { + return flag & TransactionRollbackType; + } + + + public static int resetTransactionValue(final int flag, final int type) { + return (flag & (~TransactionRollbackType)) | type; + } + + + public static int clearCompressedFlag(final int flag) { + return flag & (~CompressedFlag); + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/sysflag/PullSysFlag.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/sysflag/PullSysFlag.java new file mode 100644 index 000000000..d6c84d739 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/sysflag/PullSysFlag.java @@ -0,0 +1,77 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common.sysflag; + +/** + * Pull接口用到的flag定义 + * + * @author shijia.wxr + */ +public class PullSysFlag { + private final static int FLAG_COMMIT_OFFSET = 0x1 << 0; + private final static int FLAG_SUSPEND = 0x1 << 1; + private final static int FLAG_SUBSCRIPTION = 0x1 << 2; + private final static int FLAG_CLASS_FILTER = 0x1 << 3; + + + public static int buildSysFlag(final boolean commitOffset, final boolean suspend, + final boolean subscription, final boolean classFilter) { + int flag = 0; + + if (commitOffset) { + flag |= FLAG_COMMIT_OFFSET; + } + + if (suspend) { + flag |= FLAG_SUSPEND; + } + + if (subscription) { + flag |= FLAG_SUBSCRIPTION; + } + + if (classFilter) { + flag |= FLAG_CLASS_FILTER; + } + + return flag; + } + + + public static int clearCommitOffsetFlag(final int sysFlag) { + return sysFlag & (~FLAG_COMMIT_OFFSET); + } + + + public static boolean hasCommitOffsetFlag(final int sysFlag) { + return (sysFlag & FLAG_COMMIT_OFFSET) == FLAG_COMMIT_OFFSET; + } + + + public static boolean hasSuspendFlag(final int sysFlag) { + return (sysFlag & FLAG_SUSPEND) == FLAG_SUSPEND; + } + + + public static boolean hasSubscriptionFlag(final int sysFlag) { + return (sysFlag & FLAG_SUBSCRIPTION) == FLAG_SUBSCRIPTION; + } + + + public static boolean hasClassFilterFlag(final int sysFlag) { + return (sysFlag & FLAG_CLASS_FILTER) == FLAG_CLASS_FILTER; + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/sysflag/SubscriptionSysFlag.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/sysflag/SubscriptionSysFlag.java new file mode 100644 index 000000000..f7f19213a --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/sysflag/SubscriptionSysFlag.java @@ -0,0 +1,58 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common.sysflag; + +/** + * subscription 配置标识 + * + * @author: manhong.yqd + * @since: 14-5-28 + */ +public class SubscriptionSysFlag { + // 单元化逻辑 topic 标识 + private final static int FLAG_UNIT = 0x1 << 0; + + + public static int buildSysFlag(final boolean unit) { + int sysFlag = 0; + + if (unit) { + sysFlag |= FLAG_UNIT; + } + + return sysFlag; + } + + + public static int setUnitFlag(final int sysFlag) { + return sysFlag | FLAG_UNIT; + } + + + public static int clearUnitFlag(final int sysFlag) { + return sysFlag & (~FLAG_UNIT); + } + + + public static boolean hasUnitFlag(final int sysFlag) { + return (sysFlag & FLAG_UNIT) == FLAG_UNIT; + } + + + public static void main(String[] args) { + System.out.println(0x1 << 0); + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/sysflag/TopicSysFlag.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/sysflag/TopicSysFlag.java new file mode 100644 index 000000000..69810b576 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/sysflag/TopicSysFlag.java @@ -0,0 +1,80 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common.sysflag; + +/** + * topic 配置标识 + * + * @author: manhong.yqd + * @since: 14-5-28 + */ +public class TopicSysFlag { + // 单元化逻辑 topic 标识 + private final static int FLAG_UNIT = 0x1 << 0; + // 该 topic 有单元化订阅组 + private final static int FLAG_UNIT_SUB = 0x1 << 1; + + + public static int buildSysFlag(final boolean unit, final boolean hasUnitSub) { + int sysFlag = 0; + + if (unit) { + sysFlag |= FLAG_UNIT; + } + + if (hasUnitSub) { + sysFlag |= FLAG_UNIT_SUB; + } + + return sysFlag; + } + + + public static int setUnitFlag(final int sysFlag) { + return sysFlag | FLAG_UNIT; + } + + + public static int clearUnitFlag(final int sysFlag) { + return sysFlag & (~FLAG_UNIT); + } + + + public static boolean hasUnitFlag(final int sysFlag) { + return (sysFlag & FLAG_UNIT) == FLAG_UNIT; + } + + + public static int setUnitSubFlag(final int sysFlag) { + return sysFlag | FLAG_UNIT_SUB; + } + + + public static int clearUnitSubFlag(final int sysFlag) { + return sysFlag & (~FLAG_UNIT_SUB); + } + + + public static boolean hasUnitSubFlag(final int sysFlag) { + return (sysFlag & FLAG_UNIT_SUB) == FLAG_UNIT_SUB; + } + + + public static void main(String[] args) { + System.out.println(0x1 << 0); + System.out.println(0x1 << 1); + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/utils/HttpTinyClient.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/utils/HttpTinyClient.java new file mode 100644 index 000000000..fed66677a --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/utils/HttpTinyClient.java @@ -0,0 +1,150 @@ +package com.alibaba.rocketmq.common.utils; + +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.HttpURLConnection; +import java.net.URL; +import java.net.URLEncoder; +import java.util.Iterator; +import java.util.List; + +import com.alibaba.rocketmq.common.MQVersion; + + +/** + * HTTP 简易客户端 + * + * @author manhong.yqd + */ + +public class HttpTinyClient { + + /** + * 发送GET请求。 + */ + static public HttpResult httpGet(String url, List headers, List paramValues, + String encoding, long readTimeoutMs) throws IOException { + String encodedContent = encodingParams(paramValues, encoding); + url += (null == encodedContent) ? "" : ("?" + encodedContent); + + HttpURLConnection conn = null; + try { + conn = (HttpURLConnection) new URL(url).openConnection(); + conn.setRequestMethod("GET"); + conn.setConnectTimeout((int) readTimeoutMs); + conn.setReadTimeout((int) readTimeoutMs); + setHeaders(conn, headers, encoding); + + conn.connect(); + int respCode = conn.getResponseCode(); // 这里内部发送请求 + String resp = null; + + if (HttpURLConnection.HTTP_OK == respCode) { + resp = IOTinyUtils.toString(conn.getInputStream(), encoding); + } + else { + resp = IOTinyUtils.toString(conn.getErrorStream(), encoding); + } + return new HttpResult(respCode, resp); + } + finally { + if (conn != null) { + conn.disconnect(); + } + } + } + + + /** + * 发送POST请求。 + * + * @param url + * @param headers + * 请求Header，可以为null + * @param paramValues + * 参数，可以为null + * @param encoding + * URL编码使用的字符集 + * @param readTimeoutMs + * 响应超时 + * @return + * @throws java.io.IOException + */ + static public HttpResult httpPost(String url, List headers, List paramValues, + String encoding, long readTimeoutMs) throws IOException { + String encodedContent = encodingParams(paramValues, encoding); + + HttpURLConnection conn = null; + try { + conn = (HttpURLConnection) new URL(url).openConnection(); + conn.setRequestMethod("POST"); + conn.setConnectTimeout(3000); + conn.setReadTimeout((int) readTimeoutMs); + conn.setDoOutput(true); + conn.setDoInput(true); + setHeaders(conn, headers, encoding); + + conn.getOutputStream().write(encodedContent.getBytes()); + + int respCode = conn.getResponseCode(); // 这里内部发送请求 + String resp = null; + + if (HttpURLConnection.HTTP_OK == respCode) { + resp = IOTinyUtils.toString(conn.getInputStream(), encoding); + } + else { + resp = IOTinyUtils.toString(conn.getErrorStream(), encoding); + } + return new HttpResult(respCode, resp); + } + finally { + if (null != conn) { + conn.disconnect(); + } + } + } + + + static private void setHeaders(HttpURLConnection conn, List headers, String encoding) { + if (null != headers) { + for (Iterator iter = headers.iterator(); iter.hasNext();) { + conn.addRequestProperty(iter.next(), iter.next()); + } + } + conn.addRequestProperty("Client-Version", MQVersion.getVersionDesc(MQVersion.CurrentVersion)); + conn.addRequestProperty("Content-Type", "application/x-www-form-urlencoded;charset=" + encoding); + + // 其它 + String ts = String.valueOf(System.currentTimeMillis()); + conn.addRequestProperty("Metaq-Client-RequestTS", ts); + } + + + static private String encodingParams(List paramValues, String encoding) + throws UnsupportedEncodingException { + StringBuilder sb = new StringBuilder(); + if (null == paramValues) { + return null; + } + + for (Iterator iter = paramValues.iterator(); iter.hasNext();) { + sb.append(iter.next()).append("="); + sb.append(URLEncoder.encode(iter.next(), encoding)); + if (iter.hasNext()) { + sb.append("&"); + } + } + return sb.toString(); + } + + static public class HttpResult { + final public int code; + final public String content; + + + public HttpResult(int code, String content) { + this.code = code; + this.content = content; + } + } +} diff --git a/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/utils/IOTinyUtils.java b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/utils/IOTinyUtils.java new file mode 100644 index 000000000..7b9a03bd9 --- /dev/null +++ b/rocketmq-common/src/main/java/com/alibaba/rocketmq/common/utils/IOTinyUtils.java @@ -0,0 +1,165 @@ +package com.alibaba.rocketmq.common.utils; + +import java.io.BufferedReader; +import java.io.CharArrayWriter; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.io.Reader; +import java.io.Writer; +import java.nio.channels.FileChannel; +import java.util.ArrayList; +import java.util.List; + + +/** + * IO操作 + * + * @author manhong.yqd + * + */ +public class IOTinyUtils { + + static public String toString(InputStream input, String encoding) throws IOException { + return (null == encoding) ? toString(new InputStreamReader(input)) : toString(new InputStreamReader( + input, encoding)); + } + + + static public String toString(Reader reader) throws IOException { + CharArrayWriter sw = new CharArrayWriter(); + copy(reader, sw); + return sw.toString(); + } + + + static public long copy(Reader input, Writer output) throws IOException { + char[] buffer = new char[1 << 12]; + long count = 0; + for (int n = 0; (n = input.read(buffer)) >= 0;) { + output.write(buffer, 0, n); + count += n; + } + return count; + } + + + /** + * 从输入流读行列表。保证不返回NULL。 + */ + static public List readLines(Reader input) throws IOException { + BufferedReader reader = toBufferedReader(input); + List list = new ArrayList(); + String line = null; + for (;;) { + line = reader.readLine(); + if (null != line) { + list.add(line); + } + else { + break; + } + } + return list; + } + + + static private BufferedReader toBufferedReader(Reader reader) { + return reader instanceof BufferedReader ? (BufferedReader) reader : new BufferedReader(reader); + } + + + static public void copyFile(String source, String target) throws IOException { + File sf = new File(source); + if (!sf.exists()) { + throw new IllegalArgumentException("source file does not exist."); + } + File tf = new File(target); + tf.getParentFile().mkdirs(); + if (!tf.exists() && !tf.createNewFile()) { + throw new RuntimeException("failed to create target file."); + } + + FileChannel sc = null; + FileChannel tc = null; + try { + tc = new FileOutputStream(tf).getChannel(); + sc = new FileInputStream(sf).getChannel(); + sc.transferTo(0, sc.size(), tc); + } + finally { + if (null != sc) { + sc.close(); + } + if (null != tc) { + tc.close(); + } + } + } + + + public static void delete(File fileOrDir) throws IOException { + if (fileOrDir == null) { + return; + } + + if (fileOrDir.isDirectory()) { + cleanDirectory(fileOrDir); + } + + fileOrDir.delete(); + } + + + /** + * 清理目录下的内容 + */ + public static void cleanDirectory(File directory) throws IOException { + if (!directory.exists()) { + String message = directory + " does not exist"; + throw new IllegalArgumentException(message); + } + + if (!directory.isDirectory()) { + String message = directory + " is not a directory"; + throw new IllegalArgumentException(message); + } + + File[] files = directory.listFiles(); + if (files == null) { // null if security restricted + throw new IOException("Failed to list contents of " + directory); + } + + IOException exception = null; + for (File file : files) { + try { + delete(file); + } + catch (IOException ioe) { + exception = ioe; + } + } + + if (null != exception) { + throw exception; + } + } + + + public static void writeStringToFile(File file, String data, String encoding) throws IOException { + OutputStream os = null; + try { + os = new FileOutputStream(file); + os.write(data.getBytes(encoding)); + } + finally { + if (null != os) { + os.close(); + } + } + } +} diff --git a/rocketmq-common/src/test/java/com/alibaba/rocketmq/common/MixAllTest.java b/rocketmq-common/src/test/java/com/alibaba/rocketmq/common/MixAllTest.java new file mode 100644 index 000000000..2dfeddcf8 --- /dev/null +++ b/rocketmq-common/src/test/java/com/alibaba/rocketmq/common/MixAllTest.java @@ -0,0 +1,23 @@ +package com.alibaba.rocketmq.common; + +import java.net.InetAddress; +import java.util.List; + +import junit.framework.Assert; + +import org.junit.Test; + + +/** + * @auther lansheng.zj@taobao.com + */ +public class MixAllTest { + + @Test + public void test() throws Exception { + List localInetAddress = MixAll.getLocalInetAddress(); + String local = InetAddress.getLocalHost().getHostAddress(); + Assert.assertTrue(localInetAddress.contains("127.0.0.1")); + Assert.assertTrue(localInetAddress.contains(local)); + } +} diff --git a/rocketmq-common/src/test/java/com/alibaba/rocketmq/common/RemotingUtilTest.java b/rocketmq-common/src/test/java/com/alibaba/rocketmq/common/RemotingUtilTest.java new file mode 100644 index 000000000..30e928870 --- /dev/null +++ b/rocketmq-common/src/test/java/com/alibaba/rocketmq/common/RemotingUtilTest.java @@ -0,0 +1,29 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.common; + +import org.junit.Test; + +import com.alibaba.rocketmq.remoting.common.RemotingUtil; + + +public class RemotingUtilTest { + @Test + public void test() throws Exception { + String a = RemotingUtil.getLocalAddress(); + System.out.println(a); + } +} diff --git a/rocketmq-common/src/test/java/com/alibaba/rocketmq/common/UtilAllTest.java b/rocketmq-common/src/test/java/com/alibaba/rocketmq/common/UtilAllTest.java new file mode 100644 index 000000000..92f204958 --- /dev/null +++ b/rocketmq-common/src/test/java/com/alibaba/rocketmq/common/UtilAllTest.java @@ -0,0 +1,147 @@ +package com.alibaba.rocketmq.common; + +import static org.junit.Assert.assertTrue; + +import java.net.URL; +import java.util.Properties; + +import org.junit.Test; + + +public class UtilAllTest { + + @Test + public void test_currentStackTrace() { + System.out.println(UtilAll.currentStackTrace()); + } + + + @Test + public void test_a() { + URL url = this.getClass().getProtectionDomain().getCodeSource().getLocation(); + System.out.println(url); + System.out.println(url.getPath()); + } + + + @Test + public void test_resetClassProperties() { + DemoConfig demoConfig = new DemoConfig(); + MixAll.properties2Object(new Properties(), demoConfig); + } + + + @Test + public void test_properties2String() { + DemoConfig demoConfig = new DemoConfig(); + Properties properties = MixAll.object2Properties(demoConfig); + System.out.println(MixAll.properties2String(properties)); + } + + + @Test + public void test_timeMillisToHumanString() { + System.out.println(UtilAll.timeMillisToHumanString()); + } + + + @Test + public void test_isPropertiesEqual() { + final Properties p1 = new Properties(); + final Properties p2 = new Properties(); + + p1.setProperty("a", "1"); + p1.setProperty("b", "2"); + + p2.setProperty("a", "1"); + p2.setProperty("b", "2"); + // p2.setProperty("c", "3"); + + assertTrue(MixAll.isPropertiesEqual(p1, p2)); + } + + + @Test + public void test_getpid() { + int pid = UtilAll.getPid(); + + System.out.println("PID = " + pid); + assertTrue(pid > 0); + } + + + @Test + public void test_isBlank() { + { + boolean result = UtilAll.isBlank("Hello "); + assertTrue(!result); + } + + { + boolean result = UtilAll.isBlank(" Hello"); + assertTrue(!result); + } + + { + boolean result = UtilAll.isBlank("He llo"); + assertTrue(!result); + } + + { + boolean result = UtilAll.isBlank(" "); + assertTrue(result); + } + + { + boolean result = UtilAll.isBlank("Hello"); + assertTrue(!result); + } + } + + class DemoConfig { + private int demoWidth = 0; + private int demoLength = 0; + private boolean demoOK = false; + private String demoName = "haha"; + + + public int getDemoWidth() { + return demoWidth; + } + + + public void setDemoWidth(int demoWidth) { + this.demoWidth = demoWidth; + } + + + public int getDemoLength() { + return demoLength; + } + + + public void setDemoLength(int demoLength) { + this.demoLength = demoLength; + } + + + public boolean isDemoOK() { + return demoOK; + } + + + public void setDemoOK(boolean demoOK) { + this.demoOK = demoOK; + } + + + public String getDemoName() { + return demoName; + } + + + public void setDemoNfieldame(String demoName) { + this.demoName = demoName; + } + } +} diff --git a/rocketmq-common/src/test/java/com/alibaba/rocketmq/common/filter/FilterAPITest.java b/rocketmq-common/src/test/java/com/alibaba/rocketmq/common/filter/FilterAPITest.java new file mode 100644 index 000000000..534a3152d --- /dev/null +++ b/rocketmq-common/src/test/java/com/alibaba/rocketmq/common/filter/FilterAPITest.java @@ -0,0 +1,20 @@ +package com.alibaba.rocketmq.common.filter; + +import org.junit.Test; + +import com.alibaba.rocketmq.common.protocol.heartbeat.SubscriptionData; + + +/** + * @author shijia.wxr + * @since 2013-6-18 + */ +public class FilterAPITest { + + @Test + public void testBuildSubscriptionData() throws Exception { + SubscriptionData subscriptionData = + FilterAPI.buildSubscriptionData("ConsumerGroup1", "TestTopic", "TAG1 || Tag2 || tag3"); + System.out.println(subscriptionData); + } +} diff --git a/rocketmq-common/src/test/java/com/alibaba/rocketmq/common/filter/PolishExprTest.java b/rocketmq-common/src/test/java/com/alibaba/rocketmq/common/filter/PolishExprTest.java new file mode 100644 index 000000000..22ee24573 --- /dev/null +++ b/rocketmq-common/src/test/java/com/alibaba/rocketmq/common/filter/PolishExprTest.java @@ -0,0 +1,52 @@ +package com.alibaba.rocketmq.common.filter; + +import java.util.List; + +import junit.framework.Assert; + +import org.junit.Test; + +import com.alibaba.rocketmq.common.filter.impl.Op; +import com.alibaba.rocketmq.common.filter.impl.PolishExpr; + + +/** + * @auther lansheng.zj@taobao.com + */ +public class PolishExprTest { + + private String expression = "tag1||(tag2&&tag3)&&tag4||tag5&&(tag6 && tag7)|| tag8 && tag9"; + private PolishExpr polishExpr; + + + public void init() { + polishExpr = new PolishExpr(); + } + + + @Test + public void testReversePolish() { + List antiPolishExpression = polishExpr.reversePolish(expression); + System.out.println(antiPolishExpression); + } + + + @Test + public void testReversePolish_Performance() { + // prepare + for (int i = 0; i < 100000; i++) { + polishExpr.reversePolish(expression); + } + + long start = System.currentTimeMillis(); + for (int i = 0; i < 100000; i++) { + polishExpr.reversePolish(expression); + } + long cost = System.currentTimeMillis() - start; + System.out.println(cost); + // System.out.println(cost / 100000F); + + Assert.assertTrue(cost < 500); + } + +} diff --git a/rocketmq-common/src/test/java/com/alibaba/rocketmq/common/protocol/ConsumeStatusTest.java b/rocketmq-common/src/test/java/com/alibaba/rocketmq/common/protocol/ConsumeStatusTest.java new file mode 100644 index 000000000..4ec924a6c --- /dev/null +++ b/rocketmq-common/src/test/java/com/alibaba/rocketmq/common/protocol/ConsumeStatusTest.java @@ -0,0 +1,20 @@ +package com.alibaba.rocketmq.common.protocol; + +import org.junit.Test; + +import com.alibaba.rocketmq.common.protocol.body.ConsumeStatus; +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; + + +public class ConsumeStatusTest { + + @Test + public void decode_test() throws Exception { + ConsumeStatus cs = new ConsumeStatus(); + cs.setConsumeFailedTPS(0L); + String json = RemotingSerializable.toJson(cs, true); + System.out.println(json); + ConsumeStatus fromJson = RemotingSerializable.fromJson(json, ConsumeStatus.class); + } + +} diff --git a/rocketmq-common/src/test/java/com/alibaba/rocketmq/common/protocol/MQProtosHelperTest.java b/rocketmq-common/src/test/java/com/alibaba/rocketmq/common/protocol/MQProtosHelperTest.java new file mode 100644 index 000000000..e535ac8da --- /dev/null +++ b/rocketmq-common/src/test/java/com/alibaba/rocketmq/common/protocol/MQProtosHelperTest.java @@ -0,0 +1,8 @@ +package com.alibaba.rocketmq.common.protocol; + +/** + * @author shijia.wxr + */ +public class MQProtosHelperTest { + +} diff --git a/rocketmq-example/pom.xml b/rocketmq-example/pom.xml new file mode 100644 index 000000000..d0c32fcea --- /dev/null +++ b/rocketmq-example/pom.xml @@ -0,0 +1,41 @@ + + + com.alibaba.rocketmq + rocketmq-all + 3.2.2 + + + 4.0.0 + jar + rocketmq-example + rocketmq-example ${project.version} + + + + junit + junit + test + + + ${project.groupId} + rocketmq-client + + + ${project.groupId} + rocketmq-srvutil + + + ch.qos.logback + logback-classic + + + ch.qos.logback + logback-core + + + jboss + javassist + + + diff --git a/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/README.md b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/README.md new file mode 100644 index 000000000..239460eb4 --- /dev/null +++ b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/README.md @@ -0,0 +1,7 @@ +### 发送非顺序消息 +### 发送顺序消息 +### 发送事务消息 +### 订阅非顺序消息 +### 订阅顺序消息 +### 主动Pull消息 +### 广播方式订阅消息 diff --git a/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/benchmark/Consumer.java b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/benchmark/Consumer.java new file mode 100644 index 000000000..1bd8d4a2f --- /dev/null +++ b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/benchmark/Consumer.java @@ -0,0 +1,192 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.example.benchmark; + +import java.util.LinkedList; +import java.util.List; +import java.util.Timer; +import java.util.TimerTask; +import java.util.concurrent.atomic.AtomicLong; + +import com.alibaba.rocketmq.client.consumer.DefaultMQPushConsumer; +import com.alibaba.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext; +import com.alibaba.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus; +import com.alibaba.rocketmq.client.consumer.listener.MessageListenerConcurrently; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.common.message.MessageExt; + + +/** + * 性能测试，订阅消息 + */ +public class Consumer { + + public static void main(String[] args) throws MQClientException { + final StatsBenchmarkConsumer statsBenchmarkConsumer = new StatsBenchmarkConsumer(); + + final Timer timer = new Timer("BenchmarkTimerThread", true); + + final LinkedList snapshotList = new LinkedList(); + + timer.scheduleAtFixedRate(new TimerTask() { + @Override + public void run() { + snapshotList.addLast(statsBenchmarkConsumer.createSnapshot()); + if (snapshotList.size() > 10) { + snapshotList.removeFirst(); + } + } + }, 1000, 1000); + + timer.scheduleAtFixedRate(new TimerTask() { + private void printStats() { + if (snapshotList.size() >= 10) { + Long[] begin = snapshotList.getFirst(); + Long[] end = snapshotList.getLast(); + + final long consumeTps = + (long) (((end[1] - begin[1]) / (double) (end[0] - begin[0])) * 1000L); + final double averageB2CRT = ((end[2] - begin[2]) / (double) (end[1] - begin[1])); + final double averageS2CRT = ((end[3] - begin[3]) / (double) (end[1] - begin[1])); + + System.out.printf( + "Consume TPS: %d Average(B2C) RT: %7.3f Average(S2C) RT: %7.3f MAX(B2C) RT: %d MAX(S2C) RT: %d\n"// + , consumeTps// + , averageB2CRT// + , averageS2CRT// + , end[4]// + , end[5]// + ); + } + } + + + @Override + public void run() { + try { + this.printStats(); + } + catch (Exception e) { + e.printStackTrace(); + } + } + }, 10000, 10000); + + DefaultMQPushConsumer consumer = + new DefaultMQPushConsumer("benchmark_consumer_" + + Long.toString(System.currentTimeMillis() % 100)); + consumer.setInstanceName(Long.toString(System.currentTimeMillis())); + + consumer.subscribe("BenchmarkTest", "*"); + + consumer.registerMessageListener(new MessageListenerConcurrently() { + @Override + public ConsumeConcurrentlyStatus consumeMessage(List msgs, + ConsumeConcurrentlyContext context) { + MessageExt msg = msgs.get(0); + long now = System.currentTimeMillis(); + + // 1 + statsBenchmarkConsumer.getReceiveMessageTotalCount().incrementAndGet(); + + // 2 + long born2ConsumerRT = now - msg.getBornTimestamp(); + statsBenchmarkConsumer.getBorn2ConsumerTotalRT().addAndGet(born2ConsumerRT); + + // 3 + long store2ConsumerRT = now - msg.getStoreTimestamp(); + statsBenchmarkConsumer.getStore2ConsumerTotalRT().addAndGet(store2ConsumerRT); + + // 4 + compareAndSetMax(statsBenchmarkConsumer.getBorn2ConsumerMaxRT(), born2ConsumerRT); + + // 5 + compareAndSetMax(statsBenchmarkConsumer.getStore2ConsumerMaxRT(), store2ConsumerRT); + + return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; + } + }); + + consumer.start(); + + System.out.println("Consumer Started."); + } + + + public static void compareAndSetMax(final AtomicLong target, final long value) { + long prev = target.get(); + while (value > prev) { + boolean updated = target.compareAndSet(prev, value); + if (updated) + break; + + prev = target.get(); + } + } +} + + +class StatsBenchmarkConsumer { + // 1 + private final AtomicLong receiveMessageTotalCount = new AtomicLong(0L); + // 2 + private final AtomicLong born2ConsumerTotalRT = new AtomicLong(0L); + // 3 + private final AtomicLong store2ConsumerTotalRT = new AtomicLong(0L); + // 4 + private final AtomicLong born2ConsumerMaxRT = new AtomicLong(0L); + // 5 + private final AtomicLong store2ConsumerMaxRT = new AtomicLong(0L); + + + public Long[] createSnapshot() { + Long[] snap = new Long[] {// + System.currentTimeMillis(),// + this.receiveMessageTotalCount.get(),// + this.born2ConsumerTotalRT.get(),// + this.store2ConsumerTotalRT.get(),// + this.born2ConsumerMaxRT.get(),// + this.store2ConsumerMaxRT.get(), // + }; + + return snap; + } + + + public AtomicLong getReceiveMessageTotalCount() { + return receiveMessageTotalCount; + } + + + public AtomicLong getBorn2ConsumerTotalRT() { + return born2ConsumerTotalRT; + } + + + public AtomicLong getStore2ConsumerTotalRT() { + return store2ConsumerTotalRT; + } + + + public AtomicLong getBorn2ConsumerMaxRT() { + return born2ConsumerMaxRT; + } + + + public AtomicLong getStore2ConsumerMaxRT() { + return store2ConsumerMaxRT; + } +} diff --git a/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/benchmark/Producer.java b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/benchmark/Producer.java new file mode 100644 index 000000000..b2dfdf4d3 --- /dev/null +++ b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/benchmark/Producer.java @@ -0,0 +1,240 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.example.benchmark; + +import java.util.LinkedList; +import java.util.Timer; +import java.util.TimerTask; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicLong; + +import com.alibaba.rocketmq.client.exception.MQBrokerException; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.client.producer.DefaultMQProducer; +import com.alibaba.rocketmq.common.message.Message; +import com.alibaba.rocketmq.remoting.exception.RemotingException; + + +/** + * 性能测试，多线程同步发送消息 + */ +public class Producer { + public static void main(String[] args) throws MQClientException { + final int threadCount = args.length >= 1 ? Integer.parseInt(args[0]) : 64; + final int messageSize = args.length >= 2 ? Integer.parseInt(args[1]) : 128; + final boolean keyEnable = args.length >= 3 ? Boolean.parseBoolean(args[2]) : false; + + System.out + .printf("threadCount %d messageSize %d keyEnable %s\n", threadCount, messageSize, keyEnable); + + final Message msg = buildMessage(messageSize); + + final ExecutorService sendThreadPool = Executors.newFixedThreadPool(threadCount); + + final StatsBenchmarkProducer statsBenchmark = new StatsBenchmarkProducer(); + + final Timer timer = new Timer("BenchmarkTimerThread", true); + + final LinkedList snapshotList = new LinkedList(); + + timer.scheduleAtFixedRate(new TimerTask() { + @Override + public void run() { + snapshotList.addLast(statsBenchmark.createSnapshot()); + if (snapshotList.size() > 10) { + snapshotList.removeFirst(); + } + } + }, 1000, 1000); + + timer.scheduleAtFixedRate(new TimerTask() { + private void printStats() { + if (snapshotList.size() >= 10) { + Long[] begin = snapshotList.getFirst(); + Long[] end = snapshotList.getLast(); + + final long sendTps = + (long) (((end[3] - begin[3]) / (double) (end[0] - begin[0])) * 1000L); + final double averageRT = ((end[5] - begin[5]) / (double) (end[3] - begin[3])); + + System.out.printf( + "Send TPS: %d Max RT: %d Average RT: %7.3f Send Failed: %d Response Failed: %d\n"// + , sendTps// + , statsBenchmark.getSendMessageMaxRT().get()// + , averageRT// + , end[2]// + , end[4]// + ); + } + } + + + @Override + public void run() { + try { + this.printStats(); + } + catch (Exception e) { + e.printStackTrace(); + } + } + }, 10000, 10000); + + final DefaultMQProducer producer = new DefaultMQProducer("benchmark_producer"); + producer.setInstanceName(Long.toString(System.currentTimeMillis())); + + producer.setCompressMsgBodyOverHowmuch(Integer.MAX_VALUE); + + producer.start(); + + for (int i = 0; i < threadCount; i++) { + sendThreadPool.execute(new Runnable() { + @Override + public void run() { + while (true) { + try { + final long beginTimestamp = System.currentTimeMillis(); + if (keyEnable) { + msg.setKeys(String.valueOf(beginTimestamp / 1000)); + } + producer.send(msg); + statsBenchmark.getSendRequestSuccessCount().incrementAndGet(); + statsBenchmark.getReceiveResponseSuccessCount().incrementAndGet(); + final long currentRT = System.currentTimeMillis() - beginTimestamp; + statsBenchmark.getSendMessageSuccessTimeTotal().addAndGet(currentRT); + long prevMaxRT = statsBenchmark.getSendMessageMaxRT().get(); + while (currentRT > prevMaxRT) { + boolean updated = + statsBenchmark.getSendMessageMaxRT().compareAndSet(prevMaxRT, + currentRT); + if (updated) + break; + + prevMaxRT = statsBenchmark.getSendMessageMaxRT().get(); + } + } + catch (RemotingException e) { + statsBenchmark.getSendRequestFailedCount().incrementAndGet(); + e.printStackTrace(); + + try { + Thread.sleep(3000); + } + catch (InterruptedException e1) { + } + } + catch (InterruptedException e) { + statsBenchmark.getSendRequestFailedCount().incrementAndGet(); + try { + Thread.sleep(3000); + } + catch (InterruptedException e1) { + } + } + catch (MQClientException e) { + statsBenchmark.getSendRequestFailedCount().incrementAndGet(); + e.printStackTrace(); + } + catch (MQBrokerException e) { + statsBenchmark.getReceiveResponseFailedCount().incrementAndGet(); + try { + Thread.sleep(3000); + } + catch (InterruptedException e1) { + } + } + } + } + }); + } + } + + + private static Message buildMessage(final int messageSize) { + Message msg = new Message(); + msg.setTopic("BenchmarkTest"); + + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < messageSize; i += 10) { + sb.append("hello baby"); + } + + msg.setBody(sb.toString().getBytes()); + + return msg; + } +} + + +class StatsBenchmarkProducer { + // 1 + private final AtomicLong sendRequestSuccessCount = new AtomicLong(0L); + // 2 + private final AtomicLong sendRequestFailedCount = new AtomicLong(0L); + // 3 + private final AtomicLong receiveResponseSuccessCount = new AtomicLong(0L); + // 4 + private final AtomicLong receiveResponseFailedCount = new AtomicLong(0L); + // 5 + private final AtomicLong sendMessageSuccessTimeTotal = new AtomicLong(0L); + // 6 + private final AtomicLong sendMessageMaxRT = new AtomicLong(0L); + + + public Long[] createSnapshot() { + Long[] snap = new Long[] {// + System.currentTimeMillis(),// + this.sendRequestSuccessCount.get(),// + this.sendRequestFailedCount.get(),// + this.receiveResponseSuccessCount.get(),// + this.receiveResponseFailedCount.get(),// + this.sendMessageSuccessTimeTotal.get(), // + }; + + return snap; + } + + + public AtomicLong getSendRequestSuccessCount() { + return sendRequestSuccessCount; + } + + + public AtomicLong getSendRequestFailedCount() { + return sendRequestFailedCount; + } + + + public AtomicLong getReceiveResponseSuccessCount() { + return receiveResponseSuccessCount; + } + + + public AtomicLong getReceiveResponseFailedCount() { + return receiveResponseFailedCount; + } + + + public AtomicLong getSendMessageSuccessTimeTotal() { + return sendMessageSuccessTimeTotal; + } + + + public AtomicLong getSendMessageMaxRT() { + return sendMessageMaxRT; + } +} diff --git a/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/benchmark/TransactionProducer.java b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/benchmark/TransactionProducer.java new file mode 100644 index 000000000..7f5a2b423 --- /dev/null +++ b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/benchmark/TransactionProducer.java @@ -0,0 +1,278 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.example.benchmark; + +import java.util.LinkedList; +import java.util.Timer; +import java.util.TimerTask; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicLong; + +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.client.producer.LocalTransactionExecuter; +import com.alibaba.rocketmq.client.producer.LocalTransactionState; +import com.alibaba.rocketmq.client.producer.SendResult; +import com.alibaba.rocketmq.client.producer.TransactionCheckListener; +import com.alibaba.rocketmq.client.producer.TransactionMQProducer; +import com.alibaba.rocketmq.common.message.Message; +import com.alibaba.rocketmq.common.message.MessageExt; + + +/** + * 性能测试，多线程同步发送事务消息 + */ +public class TransactionProducer { + private static int threadCount; + private static int messageSize; + private static boolean ischeck; + private static boolean ischeckffalse; + + + public static void main(String[] args) throws MQClientException { + threadCount = args.length >= 1 ? Integer.parseInt(args[0]) : 32; + messageSize = args.length >= 2 ? Integer.parseInt(args[1]) : 1024 * 2; + ischeck = args.length >= 3 ? Boolean.parseBoolean(args[2]) : false; + ischeckffalse = args.length >= 4 ? Boolean.parseBoolean(args[3]) : false; + + final Message msg = buildMessage(messageSize); + + final ExecutorService sendThreadPool = Executors.newFixedThreadPool(threadCount); + + final StatsBenchmarkTProducer statsBenchmark = new StatsBenchmarkTProducer(); + + final Timer timer = new Timer("BenchmarkTimerThread", true); + + final LinkedList snapshotList = new LinkedList(); + + timer.scheduleAtFixedRate(new TimerTask() { + @Override + public void run() { + snapshotList.addLast(statsBenchmark.createSnapshot()); + while (snapshotList.size() > 10) { + snapshotList.removeFirst(); + } + } + }, 1000, 1000); + + timer.scheduleAtFixedRate(new TimerTask() { + private void printStats() { + if (snapshotList.size() >= 10) { + Long[] begin = snapshotList.getFirst(); + Long[] end = snapshotList.getLast(); + + final long sendTps = + (long) (((end[3] - begin[3]) / (double) (end[0] - begin[0])) * 1000L); + final double averageRT = ((end[5] - begin[5]) / (double) (end[3] - begin[3])); + + System.out.printf( + "Send TPS: %d Max RT: %d Average RT: %7.3f Send Failed: %d Response Failed: %d transaction checkCount: %d \n"// + , sendTps// + , statsBenchmark.getSendMessageMaxRT().get()// + , averageRT// + , end[2]// + , end[4]// + , end[6]); + } + } + + + @Override + public void run() { + try { + this.printStats(); + } + catch (Exception e) { + e.printStackTrace(); + } + } + }, 10000, 10000); + + final TransactionCheckListener transactionCheckListener = + new TransactionCheckListenerBImpl(ischeckffalse, statsBenchmark); + final TransactionMQProducer producer = new TransactionMQProducer("benchmark_transaction_producer"); + producer.setInstanceName(Long.toString(System.currentTimeMillis())); + producer.setTransactionCheckListener(transactionCheckListener); + producer.setDefaultTopicQueueNums(1000); + producer.start(); + + final TransactionExecuterBImpl tranExecuter = new TransactionExecuterBImpl(ischeck); + + for (int i = 0; i < threadCount; i++) { + sendThreadPool.execute(new Runnable() { + @Override + public void run() { + while (true) { + try { + // Thread.sleep(1000); + final long beginTimestamp = System.currentTimeMillis(); + SendResult sendResult = + producer.sendMessageInTransaction(msg, tranExecuter, null); + if (sendResult != null) { + statsBenchmark.getSendRequestSuccessCount().incrementAndGet(); + statsBenchmark.getReceiveResponseSuccessCount().incrementAndGet(); + } + + final long currentRT = System.currentTimeMillis() - beginTimestamp; + statsBenchmark.getSendMessageSuccessTimeTotal().addAndGet(currentRT); + long prevMaxRT = statsBenchmark.getSendMessageMaxRT().get(); + while (currentRT > prevMaxRT) { + boolean updated = + statsBenchmark.getSendMessageMaxRT().compareAndSet(prevMaxRT, + currentRT); + if (updated) + break; + + prevMaxRT = statsBenchmark.getSendMessageMaxRT().get(); + } + } + catch (MQClientException e) { + statsBenchmark.getSendRequestFailedCount().incrementAndGet(); + } + } + } + }); + } + } + + + private static Message buildMessage(final int messageSize) { + Message msg = new Message(); + msg.setTopic("BenchmarkTest"); + + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < messageSize; i += 10) { + sb.append("hello baby"); + } + + msg.setBody(sb.toString().getBytes()); + + return msg; + } +} + + +class TransactionExecuterBImpl implements LocalTransactionExecuter { + + private boolean ischeck; + + + public TransactionExecuterBImpl(boolean ischeck) { + this.ischeck = ischeck; + } + + + @Override + public LocalTransactionState executeLocalTransactionBranch(final Message msg, final Object arg) { + if (ischeck) { + return LocalTransactionState.UNKNOW; + } + return LocalTransactionState.COMMIT_MESSAGE; + } +} + + +class TransactionCheckListenerBImpl implements TransactionCheckListener { + private boolean ischeckffalse; + private StatsBenchmarkTProducer statsBenchmarkTProducer; + + + public TransactionCheckListenerBImpl(boolean ischeckffalse, + StatsBenchmarkTProducer statsBenchmarkTProducer) { + this.ischeckffalse = ischeckffalse; + this.statsBenchmarkTProducer = statsBenchmarkTProducer; + } + + + @Override + public LocalTransactionState checkLocalTransactionState(MessageExt msg) { + // System.out.println("server checking TrMsg " + msg.toString()); + statsBenchmarkTProducer.getCheckRequestSuccessCount().incrementAndGet(); + if (ischeckffalse) { + + return LocalTransactionState.ROLLBACK_MESSAGE; + } + + return LocalTransactionState.COMMIT_MESSAGE; + } +} + + +class StatsBenchmarkTProducer { + // 1 + private final AtomicLong sendRequestSuccessCount = new AtomicLong(0L); + // 2 + private final AtomicLong sendRequestFailedCount = new AtomicLong(0L); + // 3 + private final AtomicLong receiveResponseSuccessCount = new AtomicLong(0L); + // 4 + private final AtomicLong receiveResponseFailedCount = new AtomicLong(0L); + // 5 + private final AtomicLong sendMessageSuccessTimeTotal = new AtomicLong(0L); + // 6 + private final AtomicLong sendMessageMaxRT = new AtomicLong(0L); + // 7 + private final AtomicLong checkRequestSuccessCount = new AtomicLong(0L); + + + public Long[] createSnapshot() { + Long[] snap = new Long[] {// + System.currentTimeMillis(),// + this.sendRequestSuccessCount.get(),// + this.sendRequestFailedCount.get(),// + this.receiveResponseSuccessCount.get(),// + this.receiveResponseFailedCount.get(),// + this.sendMessageSuccessTimeTotal.get(), // + this.checkRequestSuccessCount.get(), }; + + return snap; + } + + + public AtomicLong getSendRequestSuccessCount() { + return sendRequestSuccessCount; + } + + + public AtomicLong getSendRequestFailedCount() { + return sendRequestFailedCount; + } + + + public AtomicLong getReceiveResponseSuccessCount() { + return receiveResponseSuccessCount; + } + + + public AtomicLong getReceiveResponseFailedCount() { + return receiveResponseFailedCount; + } + + + public AtomicLong getSendMessageSuccessTimeTotal() { + return sendMessageSuccessTimeTotal; + } + + + public AtomicLong getSendMessageMaxRT() { + return sendMessageMaxRT; + } + + + public AtomicLong getCheckRequestSuccessCount() { + return checkRequestSuccessCount; + } +} diff --git a/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/broadcast/PushConsumer.java b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/broadcast/PushConsumer.java new file mode 100644 index 000000000..f16ba6269 --- /dev/null +++ b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/broadcast/PushConsumer.java @@ -0,0 +1,63 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.example.broadcast; + +import java.util.List; + +import com.alibaba.rocketmq.client.consumer.DefaultMQPushConsumer; +import com.alibaba.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext; +import com.alibaba.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus; +import com.alibaba.rocketmq.client.consumer.listener.MessageListenerConcurrently; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.common.consumer.ConsumeFromWhere; +import com.alibaba.rocketmq.common.message.MessageExt; +import com.alibaba.rocketmq.common.protocol.heartbeat.MessageModel; + + +/** + * PushConsumer，广播方式订阅消息 + * + */ +public class PushConsumer { + + public static void main(String[] args) throws InterruptedException, MQClientException { + DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("please_rename_unique_group_name_1"); + /** + * 设置Consumer第一次启动是从队列头部开始消费还是队列尾部开始消费
+ * 如果非第一次启动，那么按照上次消费的位置继续消费 + */ + consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET); + + consumer.setMessageModel(MessageModel.BROADCASTING); + + consumer.subscribe("TopicTest", "TagA || TagC || TagD"); + + consumer.registerMessageListener(new MessageListenerConcurrently() { + + @Override + public ConsumeConcurrentlyStatus consumeMessage(List msgs, + ConsumeConcurrentlyContext context) { + System.out.println(Thread.currentThread().getName() + " Receive New Messages: " + msgs); + + return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; + } + }); + + consumer.start(); + + System.out.println("Broadcast Consumer Started."); + } +} diff --git a/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/filter/Consumer.java b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/filter/Consumer.java new file mode 100644 index 000000000..30ddebfc4 --- /dev/null +++ b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/filter/Consumer.java @@ -0,0 +1,54 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.example.filter; + +import java.util.List; + +import com.alibaba.rocketmq.client.consumer.DefaultMQPushConsumer; +import com.alibaba.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext; +import com.alibaba.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus; +import com.alibaba.rocketmq.client.consumer.listener.MessageListenerConcurrently; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.common.message.MessageExt; + + +public class Consumer { + + public static void main(String[] args) throws InterruptedException, MQClientException { + DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("ConsumerGroupNamecc4"); + /** + * 使用Java代码，在服务器做消息过滤 + */ + consumer.subscribe("TopicFilter7", MessageFilterImpl.class.getCanonicalName()); + + consumer.registerMessageListener(new MessageListenerConcurrently() { + + @Override + public ConsumeConcurrentlyStatus consumeMessage(List msgs, + ConsumeConcurrentlyContext context) { + System.out.println(Thread.currentThread().getName() + " Receive New Messages: " + msgs); + return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; + } + }); + + /** + * Consumer对象在使用之前必须要调用start初始化，初始化一次即可
+ */ + consumer.start(); + + System.out.println("Consumer Started."); + } +} diff --git a/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/filter/MessageFilterImpl.java b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/filter/MessageFilterImpl.java new file mode 100644 index 000000000..9b3ec4fd9 --- /dev/null +++ b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/filter/MessageFilterImpl.java @@ -0,0 +1,21 @@ +package com.alibaba.rocketmq.example.filter; + +import com.alibaba.rocketmq.common.filter.MessageFilter; +import com.alibaba.rocketmq.common.message.MessageExt; + + +public class MessageFilterImpl implements MessageFilter { + + @Override + public boolean match(MessageExt msg) { + String property = msg.getUserProperty("SequenceId"); + if (property != null) { + int id = Integer.parseInt(property); + if ((id % 3) == 0 && (id > 10)) { + return true; + } + } + + return false; + } +} diff --git a/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/filter/Producer.java b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/filter/Producer.java new file mode 100644 index 000000000..24e0ac19d --- /dev/null +++ b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/filter/Producer.java @@ -0,0 +1,48 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.example.filter; + +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.client.producer.DefaultMQProducer; +import com.alibaba.rocketmq.client.producer.SendResult; +import com.alibaba.rocketmq.common.message.Message; + + +public class Producer { + public static void main(String[] args) throws MQClientException, InterruptedException { + DefaultMQProducer producer = new DefaultMQProducer("ProducerGroupName"); + producer.start(); + + try { + for (int i = 0; i < 6000000; i++) { + Message msg = new Message("TopicFilter7",// topic + "TagA",// tag + "OrderID001",// key + ("Hello MetaQ").getBytes());// body + + msg.putUserProperty("SequenceId", String.valueOf(i)); + + SendResult sendResult = producer.send(msg); + System.out.println(sendResult); + } + } + catch (Exception e) { + e.printStackTrace(); + } + + producer.shutdown(); + } +} diff --git a/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/operation/Consumer.java b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/operation/Consumer.java new file mode 100644 index 000000000..b2df8d1b9 --- /dev/null +++ b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/operation/Consumer.java @@ -0,0 +1,122 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.example.operation; + +import java.util.List; +import java.util.concurrent.atomic.AtomicLong; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; +import org.apache.commons.cli.PosixParser; + +import com.alibaba.rocketmq.client.consumer.DefaultMQPushConsumer; +import com.alibaba.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext; +import com.alibaba.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus; +import com.alibaba.rocketmq.client.consumer.listener.MessageListenerConcurrently; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.common.message.MessageExt; + + +public class Consumer { + + public static CommandLine buildCommandline(String[] args) { + final Options options = new Options(); + // //////////////////////////////////////////////////// + Option opt = new Option("h", "help", false, "Print help"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("g", "consumerGroup", true, "Consumer Group Name"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("t", "topic", true, "Topic Name"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("s", "subscription", true, "subscription"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("f", "returnFailedHalf", true, "return failed result, for half message"); + opt.setRequired(true); + options.addOption(opt); + + // //////////////////////////////////////////////////// + + PosixParser parser = new PosixParser(); + HelpFormatter hf = new HelpFormatter(); + hf.setWidth(110); + CommandLine commandLine = null; + try { + commandLine = parser.parse(options, args); + if (commandLine.hasOption('h')) { + hf.printHelp("producer", options, true); + return null; + } + } + catch (ParseException e) { + hf.printHelp("producer", options, true); + return null; + } + + return commandLine; + } + + + public static void main(String[] args) throws InterruptedException, MQClientException { + CommandLine commandLine = buildCommandline(args); + if (commandLine != null) { + String group = commandLine.getOptionValue('g'); + String topic = commandLine.getOptionValue('t'); + String subscription = commandLine.getOptionValue('s'); + final String returnFailedHalf = commandLine.getOptionValue('f'); + + DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(group); + consumer.setInstanceName(Long.toString(System.currentTimeMillis())); + + consumer.subscribe(topic, subscription); + + consumer.registerMessageListener(new MessageListenerConcurrently() { + AtomicLong consumeTimes = new AtomicLong(0); + + + @Override + public ConsumeConcurrentlyStatus consumeMessage(List msgs, + ConsumeConcurrentlyContext context) { + long currentTimes = this.consumeTimes.incrementAndGet(); + + System.out.printf("%-8d %s\n", currentTimes, msgs); + + if (Boolean.parseBoolean(returnFailedHalf)) { + if ((currentTimes % 2) == 0) { + return ConsumeConcurrentlyStatus.RECONSUME_LATER; + } + } + + return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; + } + }); + + consumer.start(); + + System.out.println("Consumer Started."); + } + } +} diff --git a/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/operation/Producer.java b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/operation/Producer.java new file mode 100644 index 000000000..1f69f91da --- /dev/null +++ b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/operation/Producer.java @@ -0,0 +1,119 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.example.operation; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; +import org.apache.commons.cli.PosixParser; + +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.client.producer.DefaultMQProducer; +import com.alibaba.rocketmq.client.producer.SendResult; +import com.alibaba.rocketmq.common.message.Message; + + +/** + * Producer，发送消息，内置到安装包，方便线上进行调试定位问题 + */ +public class Producer { + + public static CommandLine buildCommandline(String[] args) { + final Options options = new Options(); + // //////////////////////////////////////////////////// + Option opt = new Option("h", "help", false, "Print help"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("g", "producerGroup", true, "Producer Group Name"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("t", "topic", true, "Topic Name"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("a", "tags", true, "Tags Name"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("k", "keys", true, "Keys Name"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("c", "msgCount", true, "Message Count"); + opt.setRequired(true); + options.addOption(opt); + + // //////////////////////////////////////////////////// + + PosixParser parser = new PosixParser(); + HelpFormatter hf = new HelpFormatter(); + hf.setWidth(110); + CommandLine commandLine = null; + try { + commandLine = parser.parse(options, args); + if (commandLine.hasOption('h')) { + hf.printHelp("producer", options, true); + return null; + } + } + catch (ParseException e) { + hf.printHelp("producer", options, true); + return null; + } + + return commandLine; + } + + + public static void main(String[] args) throws MQClientException, InterruptedException { + CommandLine commandLine = buildCommandline(args); + if (commandLine != null) { + String group = commandLine.getOptionValue('g'); + String topic = commandLine.getOptionValue('t'); + String tags = commandLine.getOptionValue('a'); + String keys = commandLine.getOptionValue('k'); + String msgCount = commandLine.getOptionValue('c'); + + DefaultMQProducer producer = new DefaultMQProducer(group); + producer.setInstanceName(Long.toString(System.currentTimeMillis())); + + producer.start(); + + for (int i = 0; i < Integer.parseInt(msgCount); i++) { + try { + Message msg = new Message(// + topic,// topic + tags,// tag + keys,// key + ("Hello RocketMQ " + i).getBytes());// body + SendResult sendResult = producer.send(msg); + + System.out.printf("%-8d %s\n", i, sendResult); + } + catch (Exception e) { + e.printStackTrace(); + Thread.sleep(1000); + } + } + + producer.shutdown(); + } + } +} diff --git a/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/ordermessage/Consumer.java b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/ordermessage/Consumer.java new file mode 100644 index 000000000..ab10da19f --- /dev/null +++ b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/ordermessage/Consumer.java @@ -0,0 +1,77 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.example.ordermessage; + +import java.util.List; +import java.util.concurrent.atomic.AtomicLong; + +import com.alibaba.rocketmq.client.consumer.DefaultMQPushConsumer; +import com.alibaba.rocketmq.client.consumer.listener.ConsumeOrderlyContext; +import com.alibaba.rocketmq.client.consumer.listener.ConsumeOrderlyStatus; +import com.alibaba.rocketmq.client.consumer.listener.MessageListenerOrderly; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.common.consumer.ConsumeFromWhere; +import com.alibaba.rocketmq.common.message.MessageExt; + + +/** + * 顺序消息消费，带事务方式（应用可控制Offset什么时候提交） + */ +public class Consumer { + + public static void main(String[] args) throws MQClientException { + DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("please_rename_unique_group_name_3"); + /** + * 设置Consumer第一次启动是从队列头部开始消费还是队列尾部开始消费
+ * 如果非第一次启动，那么按照上次消费的位置继续消费 + */ + consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET); + + consumer.subscribe("TopicTest", "TagA || TagC || TagD"); + + consumer.registerMessageListener(new MessageListenerOrderly() { + AtomicLong consumeTimes = new AtomicLong(0); + + + @Override + public ConsumeOrderlyStatus consumeMessage(List msgs, ConsumeOrderlyContext context) { + context.setAutoCommit(false); + System.out.println(Thread.currentThread().getName() + " Receive New Messages: " + msgs); + this.consumeTimes.incrementAndGet(); + if ((this.consumeTimes.get() % 2) == 0) { + return ConsumeOrderlyStatus.SUCCESS; + } + else if ((this.consumeTimes.get() % 3) == 0) { + return ConsumeOrderlyStatus.ROLLBACK; + } + else if ((this.consumeTimes.get() % 4) == 0) { + return ConsumeOrderlyStatus.COMMIT; + } + else if ((this.consumeTimes.get() % 5) == 0) { + context.setSuspendCurrentQueueTimeMillis(3000); + return ConsumeOrderlyStatus.SUSPEND_CURRENT_QUEUE_A_MOMENT; + } + + return ConsumeOrderlyStatus.SUCCESS; + } + }); + + consumer.start(); + + System.out.println("Consumer Started."); + } + +} diff --git a/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/ordermessage/Producer.java b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/ordermessage/Producer.java new file mode 100644 index 000000000..c2e7c55e5 --- /dev/null +++ b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/ordermessage/Producer.java @@ -0,0 +1,77 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.example.ordermessage; + +import java.util.List; + +import com.alibaba.rocketmq.client.exception.MQBrokerException; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.client.producer.DefaultMQProducer; +import com.alibaba.rocketmq.client.producer.MQProducer; +import com.alibaba.rocketmq.client.producer.MessageQueueSelector; +import com.alibaba.rocketmq.client.producer.SendResult; +import com.alibaba.rocketmq.common.message.Message; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.remoting.exception.RemotingException; + + +/** + * Producer，发送顺序消息 + */ +public class Producer { + public static void main(String[] args) { + try { + MQProducer producer = new DefaultMQProducer("please_rename_unique_group_name"); + + producer.start(); + + String[] tags = new String[] { "TagA", "TagB", "TagC", "TagD", "TagE" }; + + for (int i = 0; i < 100; i++) { + // 订单ID相同的消息要有序 + int orderId = i % 10; + Message msg = + new Message("TopicTestjjj", tags[i % tags.length], "KEY" + i, + ("Hello RocketMQ " + i).getBytes()); + + SendResult sendResult = producer.send(msg, new MessageQueueSelector() { + @Override + public MessageQueue select(List mqs, Message msg, Object arg) { + Integer id = (Integer) arg; + int index = id % mqs.size(); + return mqs.get(index); + } + }, orderId); + + System.out.println(sendResult); + } + + producer.shutdown(); + } + catch (MQClientException e) { + e.printStackTrace(); + } + catch (RemotingException e) { + e.printStackTrace(); + } + catch (MQBrokerException e) { + e.printStackTrace(); + } + catch (InterruptedException e) { + e.printStackTrace(); + } + } +} diff --git a/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/quickstart/Consumer.java b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/quickstart/Consumer.java new file mode 100644 index 000000000..2791d00ae --- /dev/null +++ b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/quickstart/Consumer.java @@ -0,0 +1,58 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.example.quickstart; + +import java.util.List; + +import com.alibaba.rocketmq.client.consumer.DefaultMQPushConsumer; +import com.alibaba.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext; +import com.alibaba.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus; +import com.alibaba.rocketmq.client.consumer.listener.MessageListenerConcurrently; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.common.consumer.ConsumeFromWhere; +import com.alibaba.rocketmq.common.message.MessageExt; + + +/** + * Consumer，订阅消息 + */ +public class Consumer { + + public static void main(String[] args) throws InterruptedException, MQClientException { + DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("please_rename_unique_group_name_4"); + /** + * 设置Consumer第一次启动是从队列头部开始消费还是队列尾部开始消费
+ * 如果非第一次启动，那么按照上次消费的位置继续消费 + */ + consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET); + + consumer.subscribe("TopicTest", "*"); + + consumer.registerMessageListener(new MessageListenerConcurrently() { + + @Override + public ConsumeConcurrentlyStatus consumeMessage(List msgs, + ConsumeConcurrentlyContext context) { + System.out.println(Thread.currentThread().getName() + " Receive New Messages: " + msgs); + return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; + } + }); + + consumer.start(); + + System.out.println("Consumer Started."); + } +} diff --git a/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/quickstart/Producer.java b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/quickstart/Producer.java new file mode 100644 index 000000000..a6e05b300 --- /dev/null +++ b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/quickstart/Producer.java @@ -0,0 +1,51 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.example.quickstart; + +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.client.producer.DefaultMQProducer; +import com.alibaba.rocketmq.client.producer.SendResult; +import com.alibaba.rocketmq.common.message.Message; + + +/** + * Producer，发送消息 + * + */ +public class Producer { + public static void main(String[] args) throws MQClientException, InterruptedException { + DefaultMQProducer producer = new DefaultMQProducer("please_rename_unique_group_name"); + + producer.start(); + + for (int i = 0; i < 1000; i++) { + try { + Message msg = new Message("TopicTest",// topic + "TagA",// tag + ("Hello RocketMQ " + i).getBytes()// body + ); + SendResult sendResult = producer.send(msg); + System.out.println(sendResult); + } + catch (Exception e) { + e.printStackTrace(); + Thread.sleep(1000); + } + } + + producer.shutdown(); + } +} diff --git a/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/simple/CachedQueue.java b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/simple/CachedQueue.java new file mode 100644 index 000000000..f7a6b19dc --- /dev/null +++ b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/simple/CachedQueue.java @@ -0,0 +1,15 @@ +package com.alibaba.rocketmq.example.simple; + +import java.util.TreeMap; + +import com.alibaba.rocketmq.common.message.MessageExt; + + +public class CachedQueue { + private final TreeMap msgCachedTable = new TreeMap(); + + + public TreeMap getMsgCachedTable() { + return msgCachedTable; + } +} diff --git a/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/simple/Producer.java b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/simple/Producer.java new file mode 100644 index 000000000..efb133d2e --- /dev/null +++ b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/simple/Producer.java @@ -0,0 +1,68 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.example.simple; + +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.client.producer.DefaultMQProducer; +import com.alibaba.rocketmq.client.producer.SendResult; +import com.alibaba.rocketmq.common.message.Message; + + +public class Producer { + public static void main(String[] args) throws MQClientException, InterruptedException { + /** + * 一个应用创建一个Producer，由应用来维护此对象，可以设置为全局对象或者单例
+ * 注意：ProducerGroupName需要由应用来保证唯一
+ * ProducerGroup这个概念发送普通的消息时，作用不大，但是发送分布式事务消息时，比较关键， + * 因为服务器会回查这个Group下的任意一个Producer + */ + DefaultMQProducer producer = new DefaultMQProducer("ProducerGroupName"); + + /** + * Producer对象在使用之前必须要调用start初始化，初始化一次即可
+ * 注意：切记不可以在每次发送消息时，都调用start方法 + */ + producer.start(); + + /** + * 下面这段代码表明一个Producer对象可以发送多个topic，多个tag的消息。 + * 注意：send方法是同步调用，只要不抛异常就标识成功。但是发送成功也可会有多种状态，
+ * 例如消息写入Master成功，但是Slave不成功，这种情况消息属于成功，但是对于个别应用如果对消息可靠性要求极高，
+ * 需要对这种情况做处理。另外，消息可能会存在发送失败的情况，失败重试由应用来处理。 + */ + for (int i = 0; i < 1; i++) + try { + { + Message msg = new Message("TopicTest1",// topic + "TagA",// tag + "OrderID188",// key + ("Hello MetaQ").getBytes());// body + SendResult sendResult = producer.send(msg); + System.out.println(sendResult); + } + + } + catch (Exception e) { + e.printStackTrace(); + } + + /** + * 应用退出时，要调用shutdown来清理资源，关闭网络连接，从MetaQ服务器上注销自己 + * 注意：我们建议应用在JBOSS、Tomcat等容器的退出钩子里调用shutdown方法 + */ + producer.shutdown(); + } +} diff --git a/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/simple/PullConsumer.java b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/simple/PullConsumer.java new file mode 100644 index 000000000..976e0cb69 --- /dev/null +++ b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/simple/PullConsumer.java @@ -0,0 +1,86 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.example.simple; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +import com.alibaba.rocketmq.client.consumer.DefaultMQPullConsumer; +import com.alibaba.rocketmq.client.consumer.PullResult; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.common.message.MessageQueue; + + +/** + * PullConsumer，订阅消息 + */ +public class PullConsumer { + private static final Map offseTable = new HashMap(); + + + public static void main(String[] args) throws MQClientException { + DefaultMQPullConsumer consumer = new DefaultMQPullConsumer("please_rename_unique_group_name_5"); + + consumer.start(); + + Set mqs = consumer.fetchSubscribeMessageQueues("TopicTest"); + for (MessageQueue mq : mqs) { + System.out.println("Consume from the queue: " + mq); + SINGLE_MQ: while (true) { + try { + PullResult pullResult = + consumer.pullBlockIfNotFound(mq, null, getMessageQueueOffset(mq), 32); + System.out.println(pullResult); + putMessageQueueOffset(mq, pullResult.getNextBeginOffset()); + switch (pullResult.getPullStatus()) { + case FOUND: + // TODO + break; + case NO_MATCHED_MSG: + break; + case NO_NEW_MSG: + break SINGLE_MQ; + case OFFSET_ILLEGAL: + break; + default: + break; + } + } + catch (Exception e) { + e.printStackTrace(); + } + } + } + + consumer.shutdown(); + } + + + private static void putMessageQueueOffset(MessageQueue mq, long offset) { + offseTable.put(mq, offset); + } + + + private static long getMessageQueueOffset(MessageQueue mq) { + Long offset = offseTable.get(mq); + if (offset != null) + return offset; + + return 0; + } + +} diff --git a/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/simple/PullConsumerTest.java b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/simple/PullConsumerTest.java new file mode 100644 index 000000000..1ca916018 --- /dev/null +++ b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/simple/PullConsumerTest.java @@ -0,0 +1,51 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.example.simple; + +import com.alibaba.rocketmq.client.consumer.DefaultMQPullConsumer; +import com.alibaba.rocketmq.client.consumer.PullResult; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.common.message.MessageQueue; + + +/** + * PullConsumer，订阅消息 + */ +public class PullConsumerTest { + public static void main(String[] args) throws MQClientException { + DefaultMQPullConsumer consumer = new DefaultMQPullConsumer("please_rename_unique_group_name_5"); + consumer.start(); + + try { + MessageQueue mq = new MessageQueue(); + mq.setQueueId(0); + mq.setTopic("TopicTest3"); + mq.setBrokerName("vivedeMacBook-Pro.local"); + + long offset = 26; + + long beginTime = System.currentTimeMillis(); + PullResult pullResult = consumer.pullBlockIfNotFound(mq, null, offset, 32); + System.out.println(System.currentTimeMillis() - beginTime); + System.out.println(pullResult); + } + catch (Exception e) { + e.printStackTrace(); + } + + consumer.shutdown(); + } +} diff --git a/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/simple/PullScheduleService.java b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/simple/PullScheduleService.java new file mode 100644 index 000000000..60f9fe315 --- /dev/null +++ b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/simple/PullScheduleService.java @@ -0,0 +1,58 @@ +package com.alibaba.rocketmq.example.simple; + +import com.alibaba.rocketmq.client.consumer.MQPullConsumer; +import com.alibaba.rocketmq.client.consumer.MQPullConsumerScheduleService; +import com.alibaba.rocketmq.client.consumer.PullResult; +import com.alibaba.rocketmq.client.consumer.PullTaskCallback; +import com.alibaba.rocketmq.client.consumer.PullTaskContext; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.common.protocol.heartbeat.MessageModel; + + +public class PullScheduleService { + + public static void main(String[] args) throws MQClientException { + final MQPullConsumerScheduleService scheduleService = new MQPullConsumerScheduleService("GroupName1"); + + scheduleService.setMessageModel(MessageModel.CLUSTERING); + scheduleService.registerPullTaskCallback("TopicTest1", new PullTaskCallback() { + + @Override + public void doPullTask(MessageQueue mq, PullTaskContext context) { + MQPullConsumer consumer = context.getPullConsumer(); + try { + // 获取从哪里拉取 + long offset = consumer.fetchConsumeOffset(mq, false); + if (offset < 0) + offset = 0; + + PullResult pullResult = consumer.pull(mq, "*", offset, 32); + System.out.println(offset + "\t" + mq + "\t" + pullResult); + switch (pullResult.getPullStatus()) { + case FOUND: + break; + case NO_MATCHED_MSG: + break; + case NO_NEW_MSG: + case OFFSET_ILLEGAL: + break; + default: + break; + } + + // 存储Offset，客户端每隔5s会定时刷新到Broker + consumer.updateConsumeOffset(mq, pullResult.getNextBeginOffset()); + + // 设置再过100ms后重新拉取 + context.setPullNextDelayTimeMillis(100); + } + catch (Exception e) { + e.printStackTrace(); + } + } + }); + + scheduleService.start(); + } +} diff --git a/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/simple/PushConsumer.java b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/simple/PushConsumer.java new file mode 100644 index 000000000..90d08cd90 --- /dev/null +++ b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/simple/PushConsumer.java @@ -0,0 +1,97 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.example.simple; + +import java.util.List; + +import com.alibaba.rocketmq.client.consumer.DefaultMQPushConsumer; +import com.alibaba.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext; +import com.alibaba.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus; +import com.alibaba.rocketmq.client.consumer.listener.MessageListenerConcurrently; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.common.consumer.ConsumeFromWhere; +import com.alibaba.rocketmq.common.message.MessageExt; + + +public class PushConsumer { + + /** + * 当前例子是PushConsumer用法，使用方式给用户感觉是消息从RocketMQ服务器推到了应用客户端。
+ * 但是实际PushConsumer内部是使用长轮询Pull方式从Broker拉消息，然后再回调用户Listener方法
+ */ + public static void main(String[] args) throws InterruptedException, MQClientException { + /** + * 一个应用创建一个Consumer，由应用来维护此对象，可以设置为全局对象或者单例
+ * 注意：ConsumerGroupName需要由应用来保证唯一 + */ + DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("CID_001"); + + /** + * 订阅指定topic下tags分别等于TagA或TagC或TagD + */ + consumer.subscribe("TopicTest1", "TagA || TagC || TagD"); + /** + * 订阅指定topic下所有消息
+ * 注意：一个consumer对象可以订阅多个topic + */ + consumer.subscribe("TopicTest2", "*"); + consumer.subscribe("TopicTest3", "*"); + + /** + * 设置Consumer第一次启动是从队列头部开始消费还是队列尾部开始消费
+ * 如果非第一次启动，那么按照上次消费的位置继续消费 + */ + consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET); + + consumer.registerMessageListener(new MessageListenerConcurrently() { + + /** + * 默认msgs里只有一条消息，可以通过设置consumeMessageBatchMaxSize参数来批量接收消息 + */ + @Override + public ConsumeConcurrentlyStatus consumeMessage(List msgs, + ConsumeConcurrentlyContext context) { + System.out.println(Thread.currentThread().getName() + " Receive New Messages: " + msgs); + + MessageExt msg = msgs.get(0); + if (msg.getTopic().equals("TopicTest1")) { + // 执行TopicTest1的消费逻辑 + if (msg.getTags() != null && msg.getTags().equals("TagA")) { + // 执行TagA的消费 + } + else if (msg.getTags() != null && msg.getTags().equals("TagC")) { + // 执行TagC的消费 + } + else if (msg.getTags() != null && msg.getTags().equals("TagD")) { + // 执行TagD的消费 + } + } + else if (msg.getTopic().equals("TopicTest2")) { + // 执行TopicTest2的消费逻辑 + } + + return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; + } + }); + + /** + * Consumer对象在使用之前必须要调用start初始化，初始化一次即可
+ */ + consumer.start(); + + System.out.println("Consumer Started."); + } +} diff --git a/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/simple/RandomAsyncCommit.java b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/simple/RandomAsyncCommit.java new file mode 100644 index 000000000..c603accc2 --- /dev/null +++ b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/simple/RandomAsyncCommit.java @@ -0,0 +1,46 @@ +package com.alibaba.rocketmq.example.simple; + +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; + +import com.alibaba.rocketmq.common.message.MessageExt; +import com.alibaba.rocketmq.common.message.MessageQueue; + + +public class RandomAsyncCommit { + private final ConcurrentHashMap mqCachedTable = + new ConcurrentHashMap(); + + + public void putMessages(final MessageQueue mq, final List msgs) { + CachedQueue cachedQueue = this.mqCachedTable.get(mq); + if (null == cachedQueue) { + cachedQueue = new CachedQueue(); + this.mqCachedTable.put(mq, cachedQueue); + } + for (MessageExt msg : msgs) { + cachedQueue.getMsgCachedTable().put(msg.getQueueOffset(), msg); + } + } + + + public void removeMessage(final MessageQueue mq, long offset) { + CachedQueue cachedQueue = this.mqCachedTable.get(mq); + if (null != cachedQueue) { + cachedQueue.getMsgCachedTable().remove(offset); + } + } + + + /** + * 可以被提交的Offset + */ + public long commitableOffset(final MessageQueue mq) { + CachedQueue cachedQueue = this.mqCachedTable.get(mq); + if (null != cachedQueue) { + return cachedQueue.getMsgCachedTable().firstKey(); + } + + return -1; + } +} diff --git a/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/simple/TestProducer.java b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/simple/TestProducer.java new file mode 100644 index 000000000..7b636dde4 --- /dev/null +++ b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/simple/TestProducer.java @@ -0,0 +1,75 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.example.simple; + +import com.alibaba.rocketmq.client.QueryResult; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.client.producer.DefaultMQProducer; +import com.alibaba.rocketmq.client.producer.SendResult; +import com.alibaba.rocketmq.common.message.Message; +import com.alibaba.rocketmq.common.message.MessageExt; + + +public class TestProducer { + public static void main(String[] args) throws MQClientException, InterruptedException { + /** + * 一个应用创建一个Producer，由应用来维护此对象，可以设置为全局对象或者单例
+ * 注意：ProducerGroupName需要由应用来保证唯一
+ * ProducerGroup这个概念发送普通的消息时，作用不大，但是发送分布式事务消息时，比较关键， + * 因为服务器会回查这个Group下的任意一个Producer + */ + DefaultMQProducer producer = new DefaultMQProducer("ProducerGroupName"); + /** + * Producer对象在使用之前必须要调用start初始化，初始化一次即可
+ * 注意：切记不可以在每次发送消息时，都调用start方法 + */ + producer.start(); + + /** + * 下面这段代码表明一个Producer对象可以发送多个topic，多个tag的消息。 + * 注意：send方法是同步调用，只要不抛异常就标识成功。但是发送成功也可会有多种状态，
+ * 例如消息写入Master成功，但是Slave不成功，这种情况消息属于成功，但是对于个别应用如果对消息可靠性要求极高，
+ * 需要对这种情况做处理。另外，消息可能会存在发送失败的情况，失败重试由应用来处理。 + */ + for (int i = 0; i < 1; i++) + try { + { + Message msg = new Message("TopicTest1",// topic + "TagA",// tag + "key113",// key + ("Hello MetaQ").getBytes());// body + SendResult sendResult = producer.send(msg); + System.out.println(sendResult); + + QueryResult queryMessage = + producer.queryMessage("TopicTest1", "key113", 10, 0, System.currentTimeMillis()); + for (MessageExt m : queryMessage.getMessageList()) { + System.out.println(m); + } + } + + } + catch (Exception e) { + e.printStackTrace(); + } + + /** + * 应用退出时，要调用shutdown来清理资源，关闭网络连接，从MetaQ服务器上注销自己 + * 注意：我们建议应用在JBOSS、Tomcat等容器的退出钩子里调用shutdown方法 + */ + producer.shutdown(); + } +} diff --git a/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/transaction/TransactionCheckListenerImpl.java b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/transaction/TransactionCheckListenerImpl.java new file mode 100644 index 000000000..a2cca41c6 --- /dev/null +++ b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/transaction/TransactionCheckListenerImpl.java @@ -0,0 +1,49 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.example.transaction; + +import java.util.concurrent.atomic.AtomicInteger; + +import com.alibaba.rocketmq.client.producer.LocalTransactionState; +import com.alibaba.rocketmq.client.producer.TransactionCheckListener; +import com.alibaba.rocketmq.common.message.MessageExt; + + +/** + * 未决事务，服务器回查客户端 + */ +public class TransactionCheckListenerImpl implements TransactionCheckListener { + private AtomicInteger transactionIndex = new AtomicInteger(0); + + + @Override + public LocalTransactionState checkLocalTransactionState(MessageExt msg) { + System.out.println("server checking TrMsg " + msg.toString()); + + int value = transactionIndex.getAndIncrement(); + if ((value % 6) == 0) { + throw new RuntimeException("Could not find db"); + } + else if ((value % 5) == 0) { + return LocalTransactionState.ROLLBACK_MESSAGE; + } + else if ((value % 4) == 0) { + return LocalTransactionState.COMMIT_MESSAGE; + } + + return LocalTransactionState.UNKNOW; + } +} diff --git a/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/transaction/TransactionExecuterImpl.java b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/transaction/TransactionExecuterImpl.java new file mode 100644 index 000000000..2d1f0e6b8 --- /dev/null +++ b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/transaction/TransactionExecuterImpl.java @@ -0,0 +1,48 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.example.transaction; + +import java.util.concurrent.atomic.AtomicInteger; + +import com.alibaba.rocketmq.client.producer.LocalTransactionExecuter; +import com.alibaba.rocketmq.client.producer.LocalTransactionState; +import com.alibaba.rocketmq.common.message.Message; + + +/** + * 执行本地事务 + */ +public class TransactionExecuterImpl implements LocalTransactionExecuter { + private AtomicInteger transactionIndex = new AtomicInteger(1); + + + @Override + public LocalTransactionState executeLocalTransactionBranch(final Message msg, final Object arg) { + int value = transactionIndex.getAndIncrement(); + + if (value == 0) { + throw new RuntimeException("Could not find db"); + } + else if ((value % 5) == 0) { + return LocalTransactionState.ROLLBACK_MESSAGE; + } + else if ((value % 4) == 0) { + return LocalTransactionState.COMMIT_MESSAGE; + } + + return LocalTransactionState.UNKNOW; + } +} diff --git a/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/transaction/TransactionProducer.java b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/transaction/TransactionProducer.java new file mode 100644 index 000000000..ddbcad614 --- /dev/null +++ b/rocketmq-example/src/main/java/com/alibaba/rocketmq/example/transaction/TransactionProducer.java @@ -0,0 +1,67 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.example.transaction; + +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.client.producer.SendResult; +import com.alibaba.rocketmq.client.producer.TransactionCheckListener; +import com.alibaba.rocketmq.client.producer.TransactionMQProducer; +import com.alibaba.rocketmq.common.message.Message; + + +/** + * 发送事务消息例子 + * + */ +public class TransactionProducer { + public static void main(String[] args) throws MQClientException, InterruptedException { + + TransactionCheckListener transactionCheckListener = new TransactionCheckListenerImpl(); + TransactionMQProducer producer = new TransactionMQProducer("please_rename_unique_group_name"); + // 事务回查最小并发数 + producer.setCheckThreadPoolMinSize(2); + // 事务回查最大并发数 + producer.setCheckThreadPoolMaxSize(2); + // 队列数 + producer.setCheckRequestHoldMax(2000); + producer.setTransactionCheckListener(transactionCheckListener); + producer.start(); + + String[] tags = new String[] { "TagA", "TagB", "TagC", "TagD", "TagE" }; + TransactionExecuterImpl tranExecuter = new TransactionExecuterImpl(); + for (int i = 0; i < 100; i++) { + try { + Message msg = + new Message("TopicTest", tags[i % tags.length], "KEY" + i, + ("Hello RocketMQ " + i).getBytes()); + SendResult sendResult = producer.sendMessageInTransaction(msg, tranExecuter, null); + System.out.println(sendResult); + + Thread.sleep(10); + } + catch (MQClientException e) { + e.printStackTrace(); + } + } + + for (int i = 0; i < 100000; i++) { + Thread.sleep(1000); + } + + producer.shutdown(); + + } +} diff --git a/rocketmq-example/src/main/resources/MessageFilterImpl.java b/rocketmq-example/src/main/resources/MessageFilterImpl.java new file mode 100644 index 000000000..ea6f3d891 --- /dev/null +++ b/rocketmq-example/src/main/resources/MessageFilterImpl.java @@ -0,0 +1,22 @@ +package com.alibaba.rocketmq.example.filter; + +import com.alibaba.rocketmq.common.filter.MessageFilter; +import com.alibaba.rocketmq.common.message.MessageExt; + + +public class MessageFilterImpl implements MessageFilter { + + @Override + public boolean match(MessageExt msg) { + String property = msg.getProperty("SequenceId"); + if (property != null) { + int id = Integer.parseInt(property); + if (((id % 10) == 0) && // + (id > 100)) { + return true; + } + } + + return false; + } +} diff --git a/rocketmq-filtersrv/pom.xml b/rocketmq-filtersrv/pom.xml new file mode 100644 index 000000000..454fcf12b --- /dev/null +++ b/rocketmq-filtersrv/pom.xml @@ -0,0 +1,41 @@ + + + com.alibaba.rocketmq + rocketmq-all + 3.2.2 + + + 4.0.0 + jar + rocketmq-filtersrv + rocketmq-filtersrv ${project.version} + + + + junit + junit + test + + + ${project.groupId} + rocketmq-client + + + ${project.groupId} + rocketmq-store + + + ${project.groupId} + rocketmq-srvutil + + + ch.qos.logback + logback-classic + + + ch.qos.logback + logback-core + + + diff --git a/rocketmq-filtersrv/src/main/java/com/alibaba/common/lang/ArrayUtil.java b/rocketmq-filtersrv/src/main/java/com/alibaba/common/lang/ArrayUtil.java new file mode 100644 index 000000000..408695795 --- /dev/null +++ b/rocketmq-filtersrv/src/main/java/com/alibaba/common/lang/ArrayUtil.java @@ -0,0 +1,6413 @@ +package com.alibaba.common.lang; + +import java.lang.reflect.Array; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + + +/** + * 有关数组处理的工具类。 + * + *

+ * 这个类中的每个方法都可以“安全”地处理null，而不会抛出NullPointerException。 + *

+ * + * @author Michael Zhou + * @version $Id: ArrayUtil.java 1479 2006-01-13 05:40:46Z baobao $ + */ +public class ArrayUtil { + /* + * ========================================================================== + * == + */ + /* 常量和singleton。 */ + /* + * ========================================================================== + * == + */ + + /** 空的Object数组。 */ + public static final Object[] EMPTY_OBJECT_ARRAY = new Object[0]; + + /** 空的Class数组。 */ + public static final Class[] EMPTY_CLASS_ARRAY = new Class[0]; + + /** 空的String数组。 */ + public static final String[] EMPTY_STRING_ARRAY = new String[0]; + + /** 空的long数组。 */ + public static final long[] EMPTY_LONG_ARRAY = new long[0]; + + /** 空的Long数组。 */ + public static final Long[] EMPTY_LONG_OBJECT_ARRAY = new Long[0]; + + /** 空的int数组。 */ + public static final int[] EMPTY_INT_ARRAY = new int[0]; + + /** 空的Integer数组。 */ + public static final Integer[] EMPTY_INTEGER_OBJECT_ARRAY = new Integer[0]; + + /** 空的short数组。 */ + public static final short[] EMPTY_SHORT_ARRAY = new short[0]; + + /** 空的Short数组。 */ + public static final Short[] EMPTY_SHORT_OBJECT_ARRAY = new Short[0]; + + /** 空的byte数组。 */ + public static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; + + /** 空的Byte数组。 */ + public static final Byte[] EMPTY_BYTE_OBJECT_ARRAY = new Byte[0]; + + /** 空的double数组。 */ + public static final double[] EMPTY_DOUBLE_ARRAY = new double[0]; + + /** 空的Double数组。 */ + public static final Double[] EMPTY_DOUBLE_OBJECT_ARRAY = new Double[0]; + + /** 空的float数组。 */ + public static final float[] EMPTY_FLOAT_ARRAY = new float[0]; + + /** 空的Float数组。 */ + public static final Float[] EMPTY_FLOAT_OBJECT_ARRAY = new Float[0]; + + /** 空的boolean数组。 */ + public static final boolean[] EMPTY_BOOLEAN_ARRAY = new boolean[0]; + + /** 空的Boolean数组。 */ + public static final Boolean[] EMPTY_BOOLEAN_OBJECT_ARRAY = new Boolean[0]; + + /** 空的char数组。 */ + public static final char[] EMPTY_CHAR_ARRAY = new char[0]; + + /** 空的Character数组。 */ + public static final Character[] EMPTY_CHARACTER_OBJECT_ARRAY = new Character[0]; + + /** 计算hashcode所用的常量。 */ + private static final int INITIAL_NON_ZERO_ODD_NUMBER = 17; + + /** 计算hashcode所用的常量。 */ + private static final int MULTIPLIER_NON_ZERO_ODD_NUMBER = 37; + + + /* + * ========================================================================== + * == + */ + /* 判空函数。 */ + /* */ + /* 判断一个数组是否为null或包含0个元素。 */ + /* + * ========================================================================== + * == + */ + + /** + * 检查数组是否为null或空数组[]。 + * + * 
+     * ArrayUtil.isEmpty(null)              = true
+     * ArrayUtil.isEmpty(new String[0])     = true
+     * ArrayUtil.isEmpty(new String[10])    = false
+     *
+ * + * @param array + * 要检查的数组 + * + * @return 如果为空, 则返回true + */ + public static boolean isEmpty(Object[] array) { + return ((array == null) || (array.length == 0)); + } + + + /** + * 检查数组是否为null或空数组[]。 + * + * 
+     * ArrayUtil.isEmpty(null)              = true
+     * ArrayUtil.isEmpty(new long[0])     = true
+     * ArrayUtil.isEmpty(new long[10])    = false
+     *
+ * + * @param array + * 要检查的数组 + * + * @return 如果为空, 则返回true + */ + public static boolean isEmpty(long[] array) { + return ((array == null) || (array.length == 0)); + } + + + /** + * 检查数组是否为null或空数组[]。 + * + * 
+     * ArrayUtil.isEmpty(null)              = true
+     * ArrayUtil.isEmpty(new int[0])     = true
+     * ArrayUtil.isEmpty(new int[10])    = false
+     *
+ * + * @param array + * 要检查的数组 + * + * @return 如果为空, 则返回true + */ + public static boolean isEmpty(int[] array) { + return ((array == null) || (array.length == 0)); + } + + + /** + * 检查数组是否为null或空数组[]。 + * + * 
+     * ArrayUtil.isEmpty(null)              = true
+     * ArrayUtil.isEmpty(new short[0])     = true
+     * ArrayUtil.isEmpty(new short[10])    = false
+     *
+ * + * @param array + * 要检查的数组 + * + * @return 如果为空, 则返回true + */ + public static boolean isEmpty(short[] array) { + return ((array == null) || (array.length == 0)); + } + + + /** + * 检查数组是否为null或空数组[]。 + * + * 
+     * ArrayUtil.isEmpty(null)              = true
+     * ArrayUtil.isEmpty(new byte[0])     = true
+     * ArrayUtil.isEmpty(new byte[10])    = false
+     *
+ * + * @param array + * 要检查的数组 + * + * @return 如果为空, 则返回true + */ + public static boolean isEmpty(byte[] array) { + return ((array == null) || (array.length == 0)); + } + + + /** + * 检查数组是否为null或空数组[]。 + * + * 
+     * ArrayUtil.isEmpty(null)              = true
+     * ArrayUtil.isEmpty(new double[0])     = true
+     * ArrayUtil.isEmpty(new double[10])    = false
+     *
+ * + * @param array + * 要检查的数组 + * + * @return 如果为空, 则返回true + */ + public static boolean isEmpty(double[] array) { + return ((array == null) || (array.length == 0)); + } + + + /** + * 检查数组是否为null或空数组[]。 + * + * 
+     * ArrayUtil.isEmpty(null)              = true
+     * ArrayUtil.isEmpty(new float[0])     = true
+     * ArrayUtil.isEmpty(new float[10])    = false
+     *
+ * + * @param array + * 要检查的数组 + * + * @return 如果为空, 则返回true + */ + public static boolean isEmpty(float[] array) { + return ((array == null) || (array.length == 0)); + } + + + /** + * 检查数组是否为null或空数组[]。 + * + * 
+     * ArrayUtil.isEmpty(null)              = true
+     * ArrayUtil.isEmpty(new boolean[0])     = true
+     * ArrayUtil.isEmpty(new boolean[10])    = false
+     *
+ * + * @param array + * 要检查的数组 + * + * @return 如果为空, 则返回true + */ + public static boolean isEmpty(boolean[] array) { + return ((array == null) || (array.length == 0)); + } + + + /** + * 检查数组是否为null或空数组[]。 + * + * 
+     * ArrayUtil.isEmpty(null)              = true
+     * ArrayUtil.isEmpty(new char[0])     = true
+     * ArrayUtil.isEmpty(new char[10])    = false
+     *
+ * + * @param array + * 要检查的数组 + * + * @return 如果为空, 则返回true + */ + public static boolean isEmpty(char[] array) { + return ((array == null) || (array.length == 0)); + } + + + /** + * 检查数组是否不是null和空数组[]。 + * + * 
+     * ArrayUtil.isEmpty(null)              = false
+     * ArrayUtil.isEmpty(new String[0])     = false
+     * ArrayUtil.isEmpty(new String[10])    = true
+     *
+ * + * @param array + * 要检查的数组 + * + * @return 如果不为空, 则返回true + */ + public static boolean isNotEmpty(Object[] array) { + return ((array != null) && (array.length > 0)); + } + + + /** + * 检查数组是否不是null和空数组[]。 + * + * 
+     * ArrayUtil.isEmpty(null)              = false
+     * ArrayUtil.isEmpty(new long[0])     = false
+     * ArrayUtil.isEmpty(new long[10])    = true
+     *
+ * + * @param array + * 要检查的数组 + * + * @return 如果不为空, 则返回true + */ + public static boolean isNotEmpty(long[] array) { + return ((array != null) && (array.length > 0)); + } + + + /** + * 检查数组是否不是null和空数组[]。 + * + * 
+     * ArrayUtil.isEmpty(null)              = false
+     * ArrayUtil.isEmpty(new int[0])     = false
+     * ArrayUtil.isEmpty(new int[10])    = true
+     *
+ * + * @param array + * 要检查的数组 + * + * @return 如果不为空, 则返回true + */ + public static boolean isNotEmpty(int[] array) { + return ((array != null) && (array.length > 0)); + } + + + /** + * 检查数组是否不是null和空数组[]。 + * + * 
+     * ArrayUtil.isEmpty(null)              = false
+     * ArrayUtil.isEmpty(new short[0])     = false
+     * ArrayUtil.isEmpty(new short[10])    = true
+     *
+ * + * @param array + * 要检查的数组 + * + * @return 如果不为空, 则返回true + */ + public static boolean isNotEmpty(short[] array) { + return ((array != null) && (array.length > 0)); + } + + + /** + * 检查数组是否不是null和空数组[]。 + * + * 
+     * ArrayUtil.isEmpty(null)              = false
+     * ArrayUtil.isEmpty(new byte[0])     = false
+     * ArrayUtil.isEmpty(new byte[10])    = true
+     *
+ * + * @param array + * 要检查的数组 + * + * @return 如果不为空, 则返回true + */ + public static boolean isNotEmpty(byte[] array) { + return ((array != null) && (array.length > 0)); + } + + + /** + * 检查数组是否不是null和空数组[]。 + * + * 
+     * ArrayUtil.isEmpty(null)              = false
+     * ArrayUtil.isEmpty(new double[0])     = false
+     * ArrayUtil.isEmpty(new double[10])    = true
+     *
+ * + * @param array + * 要检查的数组 + * + * @return 如果不为空, 则返回true + */ + public static boolean isNotEmpty(double[] array) { + return ((array != null) && (array.length > 0)); + } + + + /** + * 检查数组是否不是null和空数组[]。 + * + * 
+     * ArrayUtil.isEmpty(null)              = false
+     * ArrayUtil.isEmpty(new float[0])     = false
+     * ArrayUtil.isEmpty(new float[10])    = true
+     *
+ * + * @param array + * 要检查的数组 + * + * @return 如果不为空, 则返回true + */ + public static boolean isNotEmpty(float[] array) { + return ((array != null) && (array.length > 0)); + } + + + /** + * 检查数组是否不是null和空数组[]。 + * + * 
+     * ArrayUtil.isEmpty(null)              = false
+     * ArrayUtil.isEmpty(new boolean[0])     = false
+     * ArrayUtil.isEmpty(new boolean[10])    = true
+     *
+ * + * @param array + * 要检查的数组 + * + * @return 如果不为空, 则返回true + */ + public static boolean isNotEmpty(boolean[] array) { + return ((array != null) && (array.length > 0)); + } + + + /** + * 检查数组是否不是null和空数组[]。 + * + * 
+     * ArrayUtil.isEmpty(null)              = false
+     * ArrayUtil.isEmpty(new char[0])     = false
+     * ArrayUtil.isEmpty(new char[10])    = true
+     *
+ * + * @param array + * 要检查的数组 + * + * @return 如果不为空, 则返回true + */ + public static boolean isNotEmpty(char[] array) { + return ((array != null) && (array.length > 0)); + } + + + /* + * ========================================================================== + * == + */ + /* 默认值函数。 */ + /* */ + /* 当数组为null或empty时，将数组转换成指定的默认数组。 */ + /* + * ========================================================================== + * == + */ + + /** + * 如果数组是null，则返回空数组[]，否则返回数组本身。 + * + * 
+     * ArrayUtil.defaultIfNull(null)           = []
+     * ArrayUtil.defaultIfNull(new String[0])  = 数组本身
+     * ArrayUtil.defaultIfNull(new String[10]) = 数组本身
+     *
+ * + * @param array + * 要转换的数组 + * + * @return 数组本身或空数组[] + */ + public static Object[] defaultIfNull(Object[] array) { + return (array == null) ? EMPTY_OBJECT_ARRAY : array; + } + + + /** + * 如果数组是null，则返回空数组[]，否则返回数组本身。 + * + * 
+     * ArrayUtil.defaultIfNull(null)           = []
+     * ArrayUtil.defaultIfNull(new long[0])  = 数组本身
+     * ArrayUtil.defaultIfNull(new long[10]) = 数组本身
+     *
+ * + * @param array + * 要转换的数组 + * + * @return 数组本身或空数组[] + */ + public static long[] defaultIfNull(long[] array) { + return (array == null) ? EMPTY_LONG_ARRAY : array; + } + + + /** + * 如果数组是null，则返回空数组[]，否则返回数组本身。 + * + * 
+     * ArrayUtil.defaultIfNull(null)           = []
+     * ArrayUtil.defaultIfNull(new int[0])  = 数组本身
+     * ArrayUtil.defaultIfNull(new int[10]) = 数组本身
+     *
+ * + * @param array + * 要转换的数组 + * + * @return 数组本身或空数组[] + */ + public static int[] defaultIfNull(int[] array) { + return (array == null) ? EMPTY_INT_ARRAY : array; + } + + + /** + * 如果数组是null，则返回空数组[]，否则返回数组本身。 + * + * 
+     * ArrayUtil.defaultIfNull(null)           = []
+     * ArrayUtil.defaultIfNull(new short[0])  = 数组本身
+     * ArrayUtil.defaultIfNull(new short[10]) = 数组本身
+     *
+ * + * @param array + * 要转换的数组 + * + * @return 数组本身或空数组[] + */ + public static short[] defaultIfNull(short[] array) { + return (array == null) ? EMPTY_SHORT_ARRAY : array; + } + + + /** + * 如果数组是null，则返回空数组[]，否则返回数组本身。 + * + * 
+     * ArrayUtil.defaultIfNull(null)           = []
+     * ArrayUtil.defaultIfNull(new byte[0])  = 数组本身
+     * ArrayUtil.defaultIfNull(new byte[10]) = 数组本身
+     *
+ * + * @param array + * 要转换的数组 + * + * @return 数组本身或空数组[] + */ + public static byte[] defaultIfNull(byte[] array) { + return (array == null) ? EMPTY_BYTE_ARRAY : array; + } + + + /** + * 如果数组是null，则返回空数组[]，否则返回数组本身。 + * + * 
+     * ArrayUtil.defaultIfNull(null)           = []
+     * ArrayUtil.defaultIfNull(new double[0])  = 数组本身
+     * ArrayUtil.defaultIfNull(new double[10]) = 数组本身
+     *
+ * + * @param array + * 要转换的数组 + * + * @return 数组本身或空数组[] + */ + public static double[] defaultIfNull(double[] array) { + return (array == null) ? EMPTY_DOUBLE_ARRAY : array; + } + + + /** + * 如果数组是null，则返回空数组[]，否则返回数组本身。 + * + * 
+     * ArrayUtil.defaultIfNull(null)           = []
+     * ArrayUtil.defaultIfNull(new float[0])  = 数组本身
+     * ArrayUtil.defaultIfNull(new float[10]) = 数组本身
+     *
+ * + * @param array + * 要转换的数组 + * + * @return 数组本身或空数组[] + */ + public static float[] defaultIfNull(float[] array) { + return (array == null) ? EMPTY_FLOAT_ARRAY : array; + } + + + /** + * 如果数组是null，则返回空数组[]，否则返回数组本身。 + * + * 
+     * ArrayUtil.defaultIfNull(null)           = []
+     * ArrayUtil.defaultIfNull(new boolean[0])  = 数组本身
+     * ArrayUtil.defaultIfNull(new boolean[10]) = 数组本身
+     *
+ * + * @param array + * 要转换的数组 + * + * @return 数组本身或空数组[] + */ + public static boolean[] defaultIfNull(boolean[] array) { + return (array == null) ? EMPTY_BOOLEAN_ARRAY : array; + } + + + /** + * 如果数组是null，则返回空数组[]，否则返回数组本身。 + * + * 
+     * ArrayUtil.defaultIfNull(null)           = []
+     * ArrayUtil.defaultIfNull(new char[0])  = 数组本身
+     * ArrayUtil.defaultIfNull(new char[10]) = 数组本身
+     *
+ * + * @param array + * 要转换的数组 + * + * @return 数组本身或空数组[] + */ + public static char[] defaultIfNull(char[] array) { + return (array == null) ? EMPTY_CHAR_ARRAY : array; + } + + + /** + * 如果数组是null，则返回指定默认数组，否则返回数组本身。 + * + * 
+     * ArrayUtil.defaultIfNull(null, defaultArray)           = defaultArray
+     * ArrayUtil.defaultIfNull(new String[0], defaultArray)  = 数组本身
+     * ArrayUtil.defaultIfNull(new String[10], defaultArray) = 数组本身
+     *
+ * + * @param array + * 要转换的数组 + * @param defaultArray + * 默认数组 + * + * @return 数组本身或指定的默认数组 + */ + public static Object[] defaultIfNull(Object[] array, Object[] defaultArray) { + return (array == null) ? defaultArray : array; + } + + + /** + * 如果数组是null，则返回指定默认数组，否则返回数组本身。 + * + * 
+     * ArrayUtil.defaultIfNull(null, defaultArray)         = defaultArray
+     * ArrayUtil.defaultIfNull(new long[0], defaultArray)  = 数组本身
+     * ArrayUtil.defaultIfNull(new long[10], defaultArray) = 数组本身
+     *
+ * + * @param array + * 要转换的数组 + * @param defaultArray + * 默认数组 + * + * @return 数组本身或指定的默认数组 + */ + public static long[] defaultIfNull(long[] array, long[] defaultArray) { + return (array == null) ? defaultArray : array; + } + + + /** + * 如果数组是null，则返回指定默认数组，否则返回数组本身。 + * + * 
+     * ArrayUtil.defaultIfNull(null, defaultArray)        = defaultArray
+     * ArrayUtil.defaultIfNull(new int[0], defaultArray)  = 数组本身
+     * ArrayUtil.defaultIfNull(new int[10], defaultArray) = 数组本身
+     *
+ * + * @param array + * 要转换的数组 + * @param defaultArray + * 默认数组 + * + * @return 数组本身或指定的默认数组 + */ + public static int[] defaultIfNull(int[] array, int[] defaultArray) { + return (array == null) ? defaultArray : array; + } + + + /** + * 如果数组是null，则返回指定默认数组，否则返回数组本身。 + * + * 
+     * ArrayUtil.defaultIfNull(null, defaultArray)          = defaultArray
+     * ArrayUtil.defaultIfNull(new short[0], defaultArray)  = 数组本身
+     * ArrayUtil.defaultIfNull(new short[10], defaultArray) = 数组本身
+     *
+ * + * @param array + * 要转换的数组 + * @param defaultArray + * 默认数组 + * + * @return 数组本身或指定的默认数组 + */ + public static short[] defaultIfNull(short[] array, short[] defaultArray) { + return (array == null) ? defaultArray : array; + } + + + /** + * 如果数组是null，则返回指定默认数组，否则返回数组本身。 + * + * 
+     * ArrayUtil.defaultIfNull(null, defaultArray)         = defaultArray
+     * ArrayUtil.defaultIfNull(new byte[0], defaultArray)  = 数组本身
+     * ArrayUtil.defaultIfNull(new byte[10], defaultArray) = 数组本身
+     *
+ * + * @param array + * 要转换的数组 + * @param defaultArray + * 默认数组 + * + * @return 数组本身或指定的默认数组 + */ + public static byte[] defaultIfNull(byte[] array, byte[] defaultArray) { + return (array == null) ? defaultArray : array; + } + + + /** + * 如果数组是null，则返回指定默认数组，否则返回数组本身。 + * + * 
+     * ArrayUtil.defaultIfNull(null, defaultArray)         = defaultArray
+     * ArrayUtil.defaultIfNull(new double[0], defaultArray)  = 数组本身
+     * ArrayUtil.defaultIfNull(new double[10], defaultArray) = 数组本身
+     *
+ * + * @param array + * 要转换的数组 + * @param defaultArray + * 默认数组 + * + * @return 数组本身或指定的默认数组 + */ + public static double[] defaultIfNull(double[] array, double[] defaultArray) { + return (array == null) ? defaultArray : array; + } + + + /** + * 如果数组是null，则返回指定默认数组，否则返回数组本身。 + * + * 
+     * ArrayUtil.defaultIfNull(null, defaultArray)          = defaultArray
+     * ArrayUtil.defaultIfNull(new float[0], defaultArray)  = 数组本身
+     * ArrayUtil.defaultIfNull(new float[10], defaultArray) = 数组本身
+     *
+ * + * @param array + * 要转换的数组 + * @param defaultArray + * 默认数组 + * + * @return 数组本身或指定的默认数组 + */ + public static float[] defaultIfNull(float[] array, float[] defaultArray) { + return (array == null) ? defaultArray : array; + } + + + /** + * 如果数组是null，则返回指定默认数组，否则返回数组本身。 + * + * 
+     * ArrayUtil.defaultIfNull(null, defaultArray)            = defaultArray
+     * ArrayUtil.defaultIfNull(new boolean[0], defaultArray)  = 数组本身
+     * ArrayUtil.defaultIfNull(new boolean[10], defaultArray) = 数组本身
+     *
+ * + * @param array + * 要转换的数组 + * @param defaultArray + * 默认数组 + * + * @return 数组本身或指定的默认数组 + */ + public static boolean[] defaultIfNull(boolean[] array, boolean[] defaultArray) { + return (array == null) ? defaultArray : array; + } + + + /** + * 如果数组是null，则返回指定默认数组，否则返回数组本身。 + * + * 
+     * ArrayUtil.defaultIfNull(null, defaultArray)         = defaultArray
+     * ArrayUtil.defaultIfNull(new char[0], defaultArray)  = 数组本身
+     * ArrayUtil.defaultIfNull(new char[10], defaultArray) = 数组本身
+     *
+ * + * @param array + * 要转换的数组 + * @param defaultArray + * 默认数组 + * + * @return 数组本身或指定的默认数组 + */ + public static char[] defaultIfNull(char[] array, char[] defaultArray) { + return (array == null) ? defaultArray : array; + } + + + /** + * 如果数组是null，则返回指定元素类型的空数组，否则返回数组本身。 + * + * 
+     * ArrayUtil.defaultIfNull(null, String.class)           = new String[0]
+     * ArrayUtil.defaultIfNull(new String[0], String.class)  = 数组本身
+     * ArrayUtil.defaultIfNull(new String[10], String.class) = 数组本身
+     *
+ * + * @param array + * 要转换的数组 + * @param defaultComponentType + * 默认数组的元素类型 + * + * @return 数组本身或指定类型的空数组 + */ + public static Object[] defaultIfNull(Object[] array, Class defaultComponentType) { + return (array == null) ? (Object[]) Array.newInstance( + ClassUtil.getNonPrimitiveType(defaultComponentType), 0) : array; + } + + + /** + * 如果数组是null或空数组[]，则返回空数组[]，否则返回数组本身。 + * + *

+ * 此方法实际上和defaultIfNull(Object[])等效。 + * + * 

+     * ArrayUtil.defaultIfEmpty(null)           = []
+     * ArrayUtil.defaultIfEmpty(new String[0])  = 数组本身
+     * ArrayUtil.defaultIfEmpty(new String[10]) = 数组本身
+     *
+ * + *

+ * + * @param array + * 要转换的数组 + * + * @return 数组本身或空数组[] + */ + public static Object[] defaultIfEmpty(Object[] array) { + return (array == null) ? EMPTY_OBJECT_ARRAY : array; + } + + + /** + * 如果数组是null或空数组[]，则返回空数组[]，否则返回数组本身。 + * + *

+ * 此方法实际上和defaultIfNull(Object[])等效。 + * + * 

+     * ArrayUtil.defaultIfEmpty(null)           = []
+     * ArrayUtil.defaultIfEmpty(new long[0])    = 数组本身
+     * ArrayUtil.defaultIfEmpty(new long[10])   = 数组本身
+     *
+ * + *

+ * + * @param array + * 要转换的数组 + * + * @return 数组本身或空数组[] + */ + public static long[] defaultIfEmpty(long[] array) { + return (array == null) ? EMPTY_LONG_ARRAY : array; + } + + + /** + * 如果数组是null或空数组[]，则返回空数组[]，否则返回数组本身。 + * + *

+ * 此方法实际上和defaultIfNull(Object[])等效。 + * + * 

+     * ArrayUtil.defaultIfEmpty(null)          = []
+     * ArrayUtil.defaultIfEmpty(new int[0])    = 数组本身
+     * ArrayUtil.defaultIfEmpty(new int[10])   = 数组本身
+     *
+ * + *

+ * + * @param array + * 要转换的数组 + * + * @return 数组本身或空数组[] + */ + public static int[] defaultIfEmpty(int[] array) { + return (array == null) ? EMPTY_INT_ARRAY : array; + } + + + /** + * 如果数组是null或空数组[]，则返回空数组[]，否则返回数组本身。 + * + *

+ * 此方法实际上和defaultIfNull(Object[])等效。 + * + * 

+     * ArrayUtil.defaultIfEmpty(null)               = []
+     * ArrayUtil.defaultIfEmpty(new short[0])    = 数组本身
+     * ArrayUtil.defaultIfEmpty(new short[10])   = 数组本身
+     *
+ * + *

+ * + * @param array + * 要转换的数组 + * + * @return 数组本身或空数组[] + */ + public static short[] defaultIfEmpty(short[] array) { + return (array == null) ? EMPTY_SHORT_ARRAY : array; + } + + + /** + * 如果数组是null或空数组[]，则返回空数组[]，否则返回数组本身。 + * + *

+ * 此方法实际上和defaultIfNull(Object[])等效。 + * + * 

+     * ArrayUtil.defaultIfEmpty(null)           = []
+     * ArrayUtil.defaultIfEmpty(new byte[0])    = 数组本身
+     * ArrayUtil.defaultIfEmpty(new byte[10])   = 数组本身
+     *
+ * + *

+ * + * @param array + * 要转换的数组 + * + * @return 数组本身或空数组[] + */ + public static byte[] defaultIfEmpty(byte[] array) { + return (array == null) ? EMPTY_BYTE_ARRAY : array; + } + + + /** + * 如果数组是null或空数组[]，则返回空数组[]，否则返回数组本身。 + * + *

+ * 此方法实际上和defaultIfNull(Object[])等效。 + * + * 

+     * ArrayUtil.defaultIfEmpty(null)               = []
+     * ArrayUtil.defaultIfEmpty(new double[0])    = 数组本身
+     * ArrayUtil.defaultIfEmpty(new double[10])   = 数组本身
+     *
+ * + *

+ * + * @param array + * 要转换的数组 + * + * @return 数组本身或空数组[] + */ + public static double[] defaultIfEmpty(double[] array) { + return (array == null) ? EMPTY_DOUBLE_ARRAY : array; + } + + + /** + * 如果数组是null或空数组[]，则返回空数组[]，否则返回数组本身。 + * + *

+ * 此方法实际上和defaultIfNull(Object[])等效。 + * + * 

+     * ArrayUtil.defaultIfEmpty(null)               = []
+     * ArrayUtil.defaultIfEmpty(new float[0])    = 数组本身
+     * ArrayUtil.defaultIfEmpty(new float[10])   = 数组本身
+     *
+ * + *

+ * + * @param array + * 要转换的数组 + * + * @return 数组本身或空数组[] + */ + public static float[] defaultIfEmpty(float[] array) { + return (array == null) ? EMPTY_FLOAT_ARRAY : array; + } + + + /** + * 如果数组是null或空数组[]，则返回空数组[]，否则返回数组本身。 + * + *

+ * 此方法实际上和defaultIfNull(Object[])等效。 + * + * 

+     * ArrayUtil.defaultIfEmpty(null)               = []
+     * ArrayUtil.defaultIfEmpty(new boolean[0])    = 数组本身
+     * ArrayUtil.defaultIfEmpty(new boolean[10])   = 数组本身
+     *
+ * + *

+ * + * @param array + * 要转换的数组 + * + * @return 数组本身或空数组[] + */ + public static boolean[] defaultIfEmpty(boolean[] array) { + return (array == null) ? EMPTY_BOOLEAN_ARRAY : array; + } + + + /** + * 如果数组是null或空数组[]，则返回空数组[]，否则返回数组本身。 + * + *

+ * 此方法实际上和defaultIfNull(Object[])等效。 + * + * 

+     * ArrayUtil.defaultIfEmpty(null)           = []
+     * ArrayUtil.defaultIfEmpty(new char[0])    = 数组本身
+     * ArrayUtil.defaultIfEmpty(new char[10])   = 数组本身
+     *
+ * + *

+ * + * @param array + * 要转换的数组 + * + * @return 数组本身或空数组[] + */ + public static char[] defaultIfEmpty(char[] array) { + return (array == null) ? EMPTY_CHAR_ARRAY : array; + } + + + /** + * 如果数组是null或空数组[]，则返回指定默认数组，否则返回数组本身。 + * + * 
+     * ArrayUtil.defaultIfEmpty(null, defaultArray)           = defaultArray
+     * ArrayUtil.defaultIfEmpty(new String[0], defaultArray)  = defaultArray
+     * ArrayUtil.defaultIfEmpty(new String[10], defaultArray) = 数组本身
+     *
+ * + * @param array + * 要转换的数组 + * @param defaultArray + * 默认数组 + * + * @return 数组本身或指定的默认数组 + */ + public static Object[] defaultIfEmpty(Object[] array, Object[] defaultArray) { + return ((array == null) || (array.length == 0)) ? defaultArray : array; + } + + + /** + * 如果数组是null或空数组[]，则返回指定默认数组，否则返回数组本身。 + * + * 
+     * ArrayUtil.defaultIfEmpty(null, defaultArray)           = defaultArray
+     * ArrayUtil.defaultIfEmpty(new long[0], defaultArray)    = defaultArray
+     * ArrayUtil.defaultIfEmpty(new long[10], defaultArray)   = 数组本身
+     *
+ * + * @param array + * 要转换的数组 + * @param defaultArray + * 默认数组 + * + * @return 数组本身或指定的默认数组 + */ + public static long[] defaultIfEmpty(long[] array, long[] defaultArray) { + return ((array == null) || (array.length == 0)) ? defaultArray : array; + } + + + /** + * 如果数组是null或空数组[]，则返回指定默认数组，否则返回数组本身。 + * + * 
+     * ArrayUtil.defaultIfEmpty(null, defaultArray)           = defaultArray
+     * ArrayUtil.defaultIfEmpty(new int[0], defaultArray)     = defaultArray
+     * ArrayUtil.defaultIfEmpty(new int[10], defaultArray)    = 数组本身
+     *
+ * + * @param array + * 要转换的数组 + * @param defaultArray + * 默认数组 + * + * @return 数组本身或指定的默认数组 + */ + public static int[] defaultIfEmpty(int[] array, int[] defaultArray) { + return ((array == null) || (array.length == 0)) ? defaultArray : array; + } + + + /** + * 如果数组是null或空数组[]，则返回指定默认数组，否则返回数组本身。 + * + * 
+     * ArrayUtil.defaultIfEmpty(null, defaultArray)           = defaultArray
+     * ArrayUtil.defaultIfEmpty(new short[0], defaultArray)   = defaultArray
+     * ArrayUtil.defaultIfEmpty(new short[10], defaultArray)  = 数组本身
+     *
+ * + * @param array + * 要转换的数组 + * @param defaultArray + * 默认数组 + * + * @return 数组本身或指定的默认数组 + */ + public static short[] defaultIfEmpty(short[] array, short[] defaultArray) { + return ((array == null) || (array.length == 0)) ? defaultArray : array; + } + + + /** + * 如果数组是null或空数组[]，则返回指定默认数组，否则返回数组本身。 + * + * 
+     * ArrayUtil.defaultIfEmpty(null, defaultArray)           = defaultArray
+     * ArrayUtil.defaultIfEmpty(new byte[0], defaultArray)    = defaultArray
+     * ArrayUtil.defaultIfEmpty(new byte[10], defaultArray)   = 数组本身
+     *
+ * + * @param array + * 要转换的数组 + * @param defaultArray + * 默认数组 + * + * @return 数组本身或指定的默认数组 + */ + public static byte[] defaultIfEmpty(byte[] array, byte[] defaultArray) { + return ((array == null) || (array.length == 0)) ? defaultArray : array; + } + + + /** + * 如果数组是null或空数组[]，则返回指定默认数组，否则返回数组本身。 + * + * 
+     * ArrayUtil.defaultIfEmpty(null, defaultArray)           = defaultArray
+     * ArrayUtil.defaultIfEmpty(new double[0], defaultArray)  = defaultArray
+     * ArrayUtil.defaultIfEmpty(new double[10], defaultArray) = 数组本身
+     *
+ * + * @param array + * 要转换的数组 + * @param defaultArray + * 默认数组 + * + * @return 数组本身或指定的默认数组 + */ + public static double[] defaultIfEmpty(double[] array, double[] defaultArray) { + return ((array == null) || (array.length == 0)) ? defaultArray : array; + } + + + /** + * 如果数组是null或空数组[]，则返回指定默认数组，否则返回数组本身。 + * + * 
+     * ArrayUtil.defaultIfEmpty(null, defaultArray)           = defaultArray
+     * ArrayUtil.defaultIfEmpty(new float[0], defaultArray)   = defaultArray
+     * ArrayUtil.defaultIfEmpty(new float[10], defaultArray)  = 数组本身
+     *
+ * + * @param array + * 要转换的数组 + * @param defaultArray + * 默认数组 + * + * @return 数组本身或指定的默认数组 + */ + public static float[] defaultIfEmpty(float[] array, float[] defaultArray) { + return ((array == null) || (array.length == 0)) ? defaultArray : array; + } + + + /** + * 如果数组是null或空数组[]，则返回指定默认数组，否则返回数组本身。 + * + * 
+     * ArrayUtil.defaultIfEmpty(null, defaultArray)              = defaultArray
+     * ArrayUtil.defaultIfEmpty(new boolean[0], defaultArray)    = defaultArray
+     * ArrayUtil.defaultIfEmpty(new boolean[10], defaultArray)   = 数组本身
+     *
+ * + * @param array + * 要转换的数组 + * @param defaultArray + * 默认数组 + * + * @return 数组本身或指定的默认数组 + */ + public static boolean[] defaultIfEmpty(boolean[] array, boolean[] defaultArray) { + return ((array == null) || (array.length == 0)) ? defaultArray : array; + } + + + /** + * 如果数组是null或空数组[]，则返回指定默认数组，否则返回数组本身。 + * + * 
+     * ArrayUtil.defaultIfEmpty(null, defaultArray)           = defaultArray
+     * ArrayUtil.defaultIfEmpty(new char[0], defaultArray)    = defaultArray
+     * ArrayUtil.defaultIfEmpty(new char[10], defaultArray)   = 数组本身
+     *
+ * + * @param array + * 要转换的数组 + * @param defaultArray + * 默认数组 + * + * @return 数组本身或指定的默认数组 + */ + public static char[] defaultIfEmpty(char[] array, char[] defaultArray) { + return ((array == null) || (array.length == 0)) ? defaultArray : array; + } + + + /** + * 如果数组是null或空数组[]，则返回指定元素类型的空数组，否则返回数组本身。 + * + * 
+     * ArrayUtil.defaultIfNull(null, String.class)           = new String[0]
+     * ArrayUtil.defaultIfNull(new String[0], String.class)  = new String[0]
+     * ArrayUtil.defaultIfNull(new String[10], String.class) = 数组本身
+     *
+ * + * @param array + * 要转换的数组 + * @param defaultComponentType + * 默认数组的元素类型 + * + * @return 数组本身或指定类型的空数组 + */ + public static Object[] defaultIfEmpty(Object[] array, Class defaultComponentType) { + return ((array == null) || (array.length == 0)) ? (Object[]) Array.newInstance( + ClassUtil.getNonPrimitiveType(defaultComponentType), 0) : array; + } + + + /* + * ========================================================================== + * == + */ + /* 比较函数。 */ + /* */ + /* 以下方法用来比较两个数组是否完全相同，支持多维数组。 */ + /* + * ========================================================================== + * == + */ + + /** + * 递归地比较两个数组是否相同，支持多维数组。 + * + *

+ * 如果比较的对象不是数组，则此方法的结果同ObjectUtil.equals。 + *

+ * + * @param array1 + * 数组1 + * @param array2 + * 数组2 + * + * @return 如果相等, 则返回true + */ + public static boolean equals(Object array1, Object array2) { + if (array1 == array2) { + return true; + } + + if ((array1 == null) || (array2 == null)) { + return false; + } + + Class clazz = array1.getClass(); + + if (!clazz.equals(array2.getClass())) { + return false; + } + + if (!clazz.isArray()) { + return array1.equals(array2); + } + + // array1和array2为同类型的数组 + if (array1 instanceof long[]) { + long[] longArray1 = (long[]) array1; + long[] longArray2 = (long[]) array2; + + if (longArray1.length != longArray2.length) { + return false; + } + + for (int i = 0; i < longArray1.length; i++) { + if (longArray1[i] != longArray2[i]) { + return false; + } + } + + return true; + } + else if (array1 instanceof int[]) { + int[] intArray1 = (int[]) array1; + int[] intArray2 = (int[]) array2; + + if (intArray1.length != intArray2.length) { + return false; + } + + for (int i = 0; i < intArray1.length; i++) { + if (intArray1[i] != intArray2[i]) { + return false; + } + } + + return true; + } + else if (array1 instanceof short[]) { + short[] shortArray1 = (short[]) array1; + short[] shortArray2 = (short[]) array2; + + if (shortArray1.length != shortArray2.length) { + return false; + } + + for (int i = 0; i < shortArray1.length; i++) { + if (shortArray1[i] != shortArray2[i]) { + return false; + } + } + + return true; + } + else if (array1 instanceof byte[]) { + byte[] byteArray1 = (byte[]) array1; + byte[] byteArray2 = (byte[]) array2; + + if (byteArray1.length != byteArray2.length) { + return false; + } + + for (int i = 0; i < byteArray1.length; i++) { + if (byteArray1[i] != byteArray2[i]) { + return false; + } + } + + return true; + } + else if (array1 instanceof double[]) { + double[] doubleArray1 = (double[]) array1; + double[] doubleArray2 = (double[]) array2; + + if (doubleArray1.length != doubleArray2.length) { + return false; + } + + for (int i = 0; i < doubleArray1.length; i++) { + if (Double.doubleToLongBits(doubleArray1[i]) != Double.doubleToLongBits(doubleArray2[i])) { + return false; + } + } + + return true; + } + else if (array1 instanceof float[]) { + float[] floatArray1 = (float[]) array1; + float[] floatArray2 = (float[]) array2; + + if (floatArray1.length != floatArray2.length) { + return false; + } + + for (int i = 0; i < floatArray1.length; i++) { + if (Float.floatToIntBits(floatArray1[i]) != Float.floatToIntBits(floatArray2[i])) { + return false; + } + } + + return true; + } + else if (array1 instanceof boolean[]) { + boolean[] booleanArray1 = (boolean[]) array1; + boolean[] booleanArray2 = (boolean[]) array2; + + if (booleanArray1.length != booleanArray2.length) { + return false; + } + + for (int i = 0; i < booleanArray1.length; i++) { + if (booleanArray1[i] != booleanArray2[i]) { + return false; + } + } + + return true; + } + else if (array1 instanceof char[]) { + char[] charArray1 = (char[]) array1; + char[] charArray2 = (char[]) array2; + + if (charArray1.length != charArray2.length) { + return false; + } + + for (int i = 0; i < charArray1.length; i++) { + if (charArray1[i] != charArray2[i]) { + return false; + } + } + + return true; + } + else { + Object[] objectArray1 = (Object[]) array1; + Object[] objectArray2 = (Object[]) array2; + + if (objectArray1.length != objectArray2.length) { + return false; + } + + for (int i = 0; i < objectArray1.length; i++) { + if (!equals(objectArray1[i], objectArray2[i])) { + return false; + } + } + + return true; + } + } + + + /* + * ========================================================================== + * == + */ + /* Hashcode函数。 */ + /* */ + /* 以下方法用来取得数组的hash code。 */ + /* + * ========================================================================== + * == + */ + + /** + * 取得数组的hash值, 如果数组为null, 则返回0。 + * + *

+ * 如果对象不是数组，则此方法的结果同ObjectUtil.hashCode。 + *

+ * + * @param array + * 数组 + * + * @return hash值 + */ + public static int hashCode(Object array) { + if (array == null) { + return 0; + } + + if (!array.getClass().isArray()) { + return array.hashCode(); + } + + int hashCode = INITIAL_NON_ZERO_ODD_NUMBER; + + // array是数组 + if (array instanceof long[]) { + long[] longArray = (long[]) array; + + for (int i = 0; i < longArray.length; i++) { + hashCode = + (hashCode * MULTIPLIER_NON_ZERO_ODD_NUMBER) + + ((int) (longArray[i] ^ (longArray[i] >> 32))); + } + } + else if (array instanceof int[]) { + int[] intArray = (int[]) array; + + for (int i = 0; i < intArray.length; i++) { + hashCode = (hashCode * MULTIPLIER_NON_ZERO_ODD_NUMBER) + intArray[i]; + } + } + else if (array instanceof short[]) { + short[] shortArray = (short[]) array; + + for (int i = 0; i < shortArray.length; i++) { + hashCode = (hashCode * MULTIPLIER_NON_ZERO_ODD_NUMBER) + shortArray[i]; + } + } + else if (array instanceof byte[]) { + byte[] byteArray = (byte[]) array; + + for (int i = 0; i < byteArray.length; i++) { + hashCode = (hashCode * MULTIPLIER_NON_ZERO_ODD_NUMBER) + byteArray[i]; + } + } + else if (array instanceof double[]) { + double[] doubleArray = (double[]) array; + + for (int i = 0; i < doubleArray.length; i++) { + long longBits = Double.doubleToLongBits(doubleArray[i]); + + hashCode = + (hashCode * MULTIPLIER_NON_ZERO_ODD_NUMBER) + ((int) (longBits ^ (longBits >> 32))); + } + } + else if (array instanceof float[]) { + float[] floatArray = (float[]) array; + + for (int i = 0; i < floatArray.length; i++) { + hashCode = (hashCode * MULTIPLIER_NON_ZERO_ODD_NUMBER) + Float.floatToIntBits(floatArray[i]); + } + } + else if (array instanceof boolean[]) { + boolean[] booleanArray = (boolean[]) array; + + for (int i = 0; i < booleanArray.length; i++) { + hashCode = (hashCode * MULTIPLIER_NON_ZERO_ODD_NUMBER) + (booleanArray[i] ? 1 : 0); + } + } + else if (array instanceof char[]) { + char[] charArray = (char[]) array; + + for (int i = 0; i < charArray.length; i++) { + hashCode = (hashCode * MULTIPLIER_NON_ZERO_ODD_NUMBER) + charArray[i]; + } + } + else { + Object[] objectArray = (Object[]) array; + + for (int i = 0; i < objectArray.length; i++) { + hashCode = (hashCode * MULTIPLIER_NON_ZERO_ODD_NUMBER) + hashCode(objectArray[i]); + } + } + + return hashCode; + } + + + /* + * ========================================================================== + * == + */ + /* 将数组转换成集合类。 */ + /* + * ========================================================================== + * == + */ + + /** + * 将数组映射成固定长度的List，当改变这个List中的值时。数组中的相应值也被改变。 + * + *

+ * 如果输入数组为null，则返回null。 + *

+ * + *

+ * 该方法内部调用java.util.Arrays.asList + * 方法所返回的列表为指定数组的映像（固定长度），因此性能和内存占用上比toList方法更优。 + *

+ * + *

+ * 这个方法常被用于初始化，例如： + * + * 

+     * List myList = ArrayUtil.toFixedList(new String[] { "aaa", "bbb", "ccc" });
+     *
+ * + *

+ * + * @param array + * 要转换的数组 + * + * @return 以数组本身为映射的list + */ + public static List toFixedList(Object[] array) { + if (array == null) { + return null; + } + + return Arrays.asList(array); + } + + + /** + * 将数组转换成List。 + * + *

+ * 如果输入数组为null，则返回null。 + *

+ * + *

+ * 该方法返回的列表为指定数组的复本，而java.util.Arrays.asList + * 方法所返回的列表为指定数组的映像（固定长度）。 + *

+ * + *

+ * 这个方法常被用于初始化，例如： + * + * 

+     * List myList = ArrayUtil.toList(new String[] { "aaa", "bbb", "ccc" });
+     * List singleList = ArrayUtil.toList("hello"); // 返回单个元素的列表["hello"]
+     *
+ * + *

+ * + * @param array + * 要转换的数组 + * + * @return 被创建的list + */ + public static List toList(Object array) { + return toList(array, null); + } + + + /** + * 将数组转换成List。 + * + *

+ * 如果输入数组为null，则返回null。 + *

+ * + *

+ * 该方法返回的列表为指定数组的复本，而java.util.Arrays.asList + * 方法所返回的列表为指定数组的映像（固定长度）。 + *

+ * + *

+ * 这个方法常被用于初始化，例如： + * + * 

+     * List myList = ArrayUtil.toList(new String[] { "aaa", "bbb", "ccc" }, new ArrayList());
+     * List singleList = ArrayUtil.toList("hello", new ArrayList()); // 返回单个元素的列表["hello"]
+     *
+ * + *

+ * + * @param array + * 要转换的数组 + * @param list + * 要填充的列表，如果是null，则创建之 + * + * @return 被创建或填充的list + */ + public static List toList(Object array, List list) { + if (array == null) { + return list; + } + + // 非array，创建一个只有一个元素的列表 + if (!array.getClass().isArray()) { + if (list == null) { + list = new ArrayList(1); + } + + list.add(array); + } + else if (array instanceof long[]) { + long[] longArray = (long[]) array; + + if (list == null) { + list = new ArrayList(longArray.length); + } + + for (int i = 0; i < longArray.length; i++) { + list.add(new Long(longArray[i])); + } + } + else if (array instanceof int[]) { + int[] intArray = (int[]) array; + + if (list == null) { + list = new ArrayList(intArray.length); + } + + for (int i = 0; i < intArray.length; i++) { + list.add(new Integer(intArray[i])); + } + } + else if (array instanceof short[]) { + short[] shortArray = (short[]) array; + + if (list == null) { + list = new ArrayList(shortArray.length); + } + + for (int i = 0; i < shortArray.length; i++) { + list.add(new Short(shortArray[i])); + } + } + else if (array instanceof byte[]) { + byte[] byteArray = (byte[]) array; + + if (list == null) { + list = new ArrayList(byteArray.length); + } + + for (int i = 0; i < byteArray.length; i++) { + list.add(new Byte(byteArray[i])); + } + } + else if (array instanceof double[]) { + double[] doubleArray = (double[]) array; + + if (list == null) { + list = new ArrayList(doubleArray.length); + } + + for (int i = 0; i < doubleArray.length; i++) { + list.add(new Double(doubleArray[i])); + } + } + else if (array instanceof float[]) { + float[] floatArray = (float[]) array; + + if (list == null) { + list = new ArrayList(floatArray.length); + } + + for (int i = 0; i < floatArray.length; i++) { + list.add(new Float(floatArray[i])); + } + } + else if (array instanceof boolean[]) { + boolean[] booleanArray = (boolean[]) array; + + if (list == null) { + list = new ArrayList(booleanArray.length); + } + + for (int i = 0; i < booleanArray.length; i++) { + list.add(booleanArray[i] ? Boolean.TRUE : Boolean.FALSE); + } + } + else if (array instanceof char[]) { + char[] charArray = (char[]) array; + + if (list == null) { + list = new ArrayList(charArray.length); + } + + for (int i = 0; i < charArray.length; i++) { + list.add(new Character(charArray[i])); + } + } + else { + Object[] objectArray = (Object[]) array; + + if (list == null) { + list = new ArrayList(objectArray.length); + } + + for (int i = 0; i < objectArray.length; i++) { + list.add(objectArray[i]); + } + } + + return list; + } + + + /** + * 将数组转换成Map。数组的元素必须是Map.Entry或元素个数多于2的子数组。 + * + *

+ * 如果输入数组为null，则返回null。 + *

+ * + *

+ * 这个方法常被用于初始化，例如： + * + * 

+     * Map colorMap = ArrayUtil.toMap(new String[][] { { "RED", "#FF0000" }, { "GREEN", "#00FF00" },
+     *                                                { "BLUE", "#0000FF" } });
+     *
+ * + *

+ * + * @param array + * 要转换的数组 + * + * @return 被创建的map + * + * @throws IllegalArgumentException + * 如果有一个子数组元素个数小于2或不是Map.Entry实例 + */ + public static Map toMap(Object[] array) { + return toMap(array, null); + } + + + /** + * 将数组转换成Map。数组的元素必须是Map.Entry或元素个数多于2的子数组。 + * + *

+ * 如果输入数组为null，则返回null。 + *

+ * + *

+ * 这个方法常被用于初始化，例如： + * + * 

+     * Map colorMap = ArrayUtil.toMap(new String[][] {{
+     *     {"RED", "#FF0000"},
+     *     {"GREEN", "#00FF00"},
+     *     {"BLUE", "#0000FF"}}, new HashMap());
+     *
+ * + *

+ * + * @param array + * 要转换的数组 + * @param map + * 要填充的map，如果为null则自动创建之 + * + * @return 被创建或填充的map + * + * @throws IllegalArgumentException + * 如果有一个子数组元素个数小于2或不是Map.Entry实例 + */ + public static Map toMap(Object[] array, Map map) { + if (array == null) { + return map; + } + + if (map == null) { + map = new HashMap((int) (array.length * 1.5)); + } + + for (int i = 0; i < array.length; i++) { + Object object = array[i]; + + if (object instanceof Map.Entry) { + Map.Entry entry = (Map.Entry) object; + + map.put(entry.getKey(), entry.getValue()); + } + else if (object instanceof Object[]) { + Object[] entry = (Object[]) object; + + if (entry.length < 2) { + throw new IllegalArgumentException("Array element " + i + ", '" + object + + "', has a length less than 2"); + } + + map.put(entry[0], entry[1]); + } + else { + throw new IllegalArgumentException("Array element " + i + ", '" + object + + "', is neither of type Map.Entry nor an Array"); + } + } + + return map; + } + + + /* + * ========================================================================== + * == + */ + /* Clone函数。 */ + /* */ + /* 以下方法调用Object.clone方法，进行“浅复制”（shallow copy）。 */ + /* + * ========================================================================== + * == + */ + + /** + * 复制一个数组。如果数组为null，则返回null。 + * + *

+ * 此方法只进行“浅复制”，也就是说，数组中的对象本身不会被复制。 另外，此方法也不处理多维数组。 + *

+ * + * @param array + * 要复制的数组 + * + * @return 数组的复本，如果原始数组为null，则返回null + */ + public static Object[] clone(Object[] array) { + if (array == null) { + return null; + } + + return (Object[]) array.clone(); + } + + + /** + * 复制一个数组。如果数组为null，则返回null。 + * + *

+ * 此方法也不处理多维数组。 + *

+ * + * @param array + * 要复制的数组 + * + * @return 数组的复本，如果原始数组为null，则返回null + */ + public static long[] clone(long[] array) { + if (array == null) { + return null; + } + + return (long[]) array.clone(); + } + + + /** + * 复制一个数组。如果数组为null，则返回null。 + * + *

+ * 此方法也不处理多维数组。 + *

+ * + * @param array + * 要复制的数组 + * + * @return 数组的复本，如果原始数组为null，则返回null + */ + public static int[] clone(int[] array) { + if (array == null) { + return null; + } + + return (int[]) array.clone(); + } + + + /** + * 复制一个数组。如果数组为null，则返回null。 + * + *

+ * 此方法也不处理多维数组。 + *

+ * + * @param array + * 要复制的数组 + * + * @return 数组的复本，如果原始数组为null，则返回null + */ + public static short[] clone(short[] array) { + if (array == null) { + return null; + } + + return (short[]) array.clone(); + } + + + /** + * 复制一个数组。如果数组为null，则返回null。 + * + *

+ * 此方法也不处理多维数组。 + *

+ * + * @param array + * 要复制的数组 + * + * @return 数组的复本，如果原始数组为null，则返回null + */ + public static byte[] clone(byte[] array) { + if (array == null) { + return null; + } + + return (byte[]) array.clone(); + } + + + /** + * 复制一个数组。如果数组为null，则返回null。 + * + *

+ * 此方法也不处理多维数组。 + *

+ * + * @param array + * 要复制的数组 + * + * @return 数组的复本，如果原始数组为null，则返回null + */ + public static double[] clone(double[] array) { + if (array == null) { + return null; + } + + return (double[]) array.clone(); + } + + + /** + * 复制一个数组。如果数组为null，则返回null。 + * + *

+ * 此方法也不处理多维数组。 + *

+ * + * @param array + * 要复制的数组 + * + * @return 数组的复本，如果原始数组为null，则返回null + */ + public static float[] clone(float[] array) { + if (array == null) { + return null; + } + + return (float[]) array.clone(); + } + + + /** + * 复制一个数组。如果数组为null，则返回null。 + * + *

+ * 此方法也不处理多维数组。 + *

+ * + * @param array + * 要复制的数组 + * + * @return 数组的复本，如果原始数组为null，则返回null + */ + public static boolean[] clone(boolean[] array) { + if (array == null) { + return null; + } + + return (boolean[]) array.clone(); + } + + + /** + * 复制一个数组。如果数组为null，则返回null。 + * + *

+ * 此方法也不处理多维数组。 + *

+ * + * @param array + * 要复制的数组 + * + * @return 数组的复本，如果原始数组为null，则返回null + */ + public static char[] clone(char[] array) { + if (array == null) { + return null; + } + + return (char[]) array.clone(); + } + + + /* + * ========================================================================== + * == + */ + /* 比较数组的长度。 */ + /* + * ========================================================================== + * == + */ + + /** + * 判断两个数组是否具有相同的长度。如果数组为null则被看作长度为0。 + * + * @param array1 + * 数组1 + * @param array2 + * 数组2 + * + * @return 如果两个数组长度相同，则返回true + */ + public static boolean isSameLength(Object[] array1, Object[] array2) { + int length1 = (array1 == null) ? 0 : array1.length; + int length2 = (array2 == null) ? 0 : array2.length; + + return length1 == length2; + } + + + /** + * 判断两个数组是否具有相同的长度。如果数组为null则被看作长度为0。 + * + * @param array1 + * 数组1 + * @param array2 + * 数组2 + * + * @return 如果两个数组长度相同，则返回true + */ + public static boolean isSameLength(long[] array1, long[] array2) { + int length1 = (array1 == null) ? 0 : array1.length; + int length2 = (array2 == null) ? 0 : array2.length; + + return length1 == length2; + } + + + /** + * 判断两个数组是否具有相同的长度。如果数组为null则被看作长度为0。 + * + * @param array1 + * 数组1 + * @param array2 + * 数组2 + * + * @return 如果两个数组长度相同，则返回true + */ + public static boolean isSameLength(int[] array1, int[] array2) { + int length1 = (array1 == null) ? 0 : array1.length; + int length2 = (array2 == null) ? 0 : array2.length; + + return length1 == length2; + } + + + /** + * 判断两个数组是否具有相同的长度。如果数组为null则被看作长度为0。 + * + * @param array1 + * 数组1 + * @param array2 + * 数组2 + * + * @return 如果两个数组长度相同，则返回true + */ + public static boolean isSameLength(short[] array1, short[] array2) { + int length1 = (array1 == null) ? 0 : array1.length; + int length2 = (array2 == null) ? 0 : array2.length; + + return length1 == length2; + } + + + /** + * 判断两个数组是否具有相同的长度。如果数组为null则被看作长度为0。 + * + * @param array1 + * 数组1 + * @param array2 + * 数组2 + * + * @return 如果两个数组长度相同，则返回true + */ + public static boolean isSameLength(byte[] array1, byte[] array2) { + int length1 = (array1 == null) ? 0 : array1.length; + int length2 = (array2 == null) ? 0 : array2.length; + + return length1 == length2; + } + + + /** + * 判断两个数组是否具有相同的长度。如果数组为null则被看作长度为0。 + * + * @param array1 + * 数组1 + * @param array2 + * 数组2 + * + * @return 如果两个数组长度相同，则返回true + */ + public static boolean isSameLength(double[] array1, double[] array2) { + int length1 = (array1 == null) ? 0 : array1.length; + int length2 = (array2 == null) ? 0 : array2.length; + + return length1 == length2; + } + + + /** + * 判断两个数组是否具有相同的长度。如果数组为null则被看作长度为0。 + * + * @param array1 + * 数组1 + * @param array2 + * 数组2 + * + * @return 如果两个数组长度相同，则返回true + */ + public static boolean isSameLength(float[] array1, float[] array2) { + int length1 = (array1 == null) ? 0 : array1.length; + int length2 = (array2 == null) ? 0 : array2.length; + + return length1 == length2; + } + + + /** + * 判断两个数组是否具有相同的长度。如果数组为null则被看作长度为0。 + * + * @param array1 + * 数组1 + * @param array2 + * 数组2 + * + * @return 如果两个数组长度相同，则返回true + */ + public static boolean isSameLength(boolean[] array1, boolean[] array2) { + int length1 = (array1 == null) ? 0 : array1.length; + int length2 = (array2 == null) ? 0 : array2.length; + + return length1 == length2; + } + + + /** + * 判断两个数组是否具有相同的长度。如果数组为null则被看作长度为0。 + * + * @param array1 + * 数组1 + * @param array2 + * 数组2 + * + * @return 如果两个数组长度相同，则返回true + */ + public static boolean isSameLength(char[] array1, char[] array2) { + int length1 = (array1 == null) ? 0 : array1.length; + int length2 = (array2 == null) ? 0 : array2.length; + + return length1 == length2; + } + + + /* + * ========================================================================== + * == + */ + /* 反转数组的元素顺序。 */ + /* + * ========================================================================== + * == + */ + + /** + * 反转数组的元素顺序。如果数组为null，则什么也不做。 + * + * @param array + * 要反转的数组 + */ + public static void reverse(Object[] array) { + if (array == null) { + return; + } + + Object tmp; + + for (int i = 0, j = array.length - 1; j > i; i++, j--) { + tmp = array[j]; + array[j] = array[i]; + array[i] = tmp; + } + } + + + /** + * 反转数组的元素顺序。如果数组为null，则什么也不做。 + * + * @param array + * 要反转的数组 + */ + public static void reverse(long[] array) { + if (array == null) { + return; + } + + long tmp; + + for (int i = 0, j = array.length - 1; j > i; i++, j--) { + tmp = array[j]; + array[j] = array[i]; + array[i] = tmp; + } + } + + + /** + * 反转数组的元素顺序。如果数组为null，则什么也不做。 + * + * @param array + * 要反转的数组 + */ + public static void reverse(int[] array) { + if (array == null) { + return; + } + + int tmp; + + for (int i = 0, j = array.length - 1; j > i; i++, j--) { + tmp = array[j]; + array[j] = array[i]; + array[i] = tmp; + } + } + + + /** + * 反转数组的元素顺序。如果数组为null，则什么也不做。 + * + * @param array + * 要反转的数组 + */ + public static void reverse(short[] array) { + if (array == null) { + return; + } + + short tmp; + + for (int i = 0, j = array.length - 1; j > i; i++, j--) { + tmp = array[j]; + array[j] = array[i]; + array[i] = tmp; + } + } + + + /** + * 反转数组的元素顺序。如果数组为null，则什么也不做。 + * + * @param array + * 要反转的数组 + */ + public static void reverse(byte[] array) { + if (array == null) { + return; + } + + byte tmp; + + for (int i = 0, j = array.length - 1; j > i; i++, j--) { + tmp = array[j]; + array[j] = array[i]; + array[i] = tmp; + } + } + + + /** + * 反转数组的元素顺序。如果数组为null，则什么也不做。 + * + * @param array + * 要反转的数组 + */ + public static void reverse(double[] array) { + if (array == null) { + return; + } + + double tmp; + + for (int i = 0, j = array.length - 1; j > i; i++, j--) { + tmp = array[j]; + array[j] = array[i]; + array[i] = tmp; + } + } + + + /** + * 反转数组的元素顺序。如果数组为null，则什么也不做。 + * + * @param array + * 要反转的数组 + */ + public static void reverse(float[] array) { + if (array == null) { + return; + } + + float tmp; + + for (int i = 0, j = array.length - 1; j > i; i++, j--) { + tmp = array[j]; + array[j] = array[i]; + array[i] = tmp; + } + } + + + /** + * 反转数组的元素顺序。如果数组为null，则什么也不做。 + * + * @param array + * 要反转的数组 + */ + public static void reverse(boolean[] array) { + if (array == null) { + return; + } + + boolean tmp; + + for (int i = 0, j = array.length - 1; j > i; i++, j--) { + tmp = array[j]; + array[j] = array[i]; + array[i] = tmp; + } + } + + + /** + * 反转数组的元素顺序。如果数组为null，则什么也不做。 + * + * @param array + * 要反转的数组 + */ + public static void reverse(char[] array) { + if (array == null) { + return; + } + + char tmp; + + for (int i = 0, j = array.length - 1; j > i; i++, j--) { + tmp = array[j]; + array[j] = array[i]; + array[i] = tmp; + } + } + + + /* + * ========================================================================== + * == + */ + /* 在数组中查找一个元素或一个元素序列。 */ + /* */ + /* 类型：Object[] */ + /* + * ========================================================================== + * == + */ + + /** + * 在数组中查找一个元素。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + * @param array + * 要扫描的数组 + * @param objectToFind + * 要查找的元素 + * + * @return 该元素在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int indexOf(Object[] array, Object objectToFind) { + return indexOf(array, objectToFind, 0); + } + + + /** + * 在数组中查找一个元素序列。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + * @param array + * 要扫描的数组 + * @param arrayToFind + * 要查找的元素序列 + * + * @return 该元素序列在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int indexOf(Object[] array, Object[] arrayToFind) { + return indexOf(array, arrayToFind, 0); + } + + + /** + * 在数组中查找一个元素。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + *

+ * 起始索引小于0则看作0，超出数组长度的起始索引则返回-1。 + *

+ * + * @param array + * 要扫描的数组 + * @param objectToFind + * 要查找的元素 + * @param startIndex + * 起始索引 + * + * @return 该元素在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int indexOf(Object[] array, Object objectToFind, int startIndex) { + if (array == null) { + return -1; + } + + if (startIndex < 0) { + startIndex = 0; + } + + if (objectToFind == null) { + for (int i = startIndex; i < array.length; i++) { + if (array[i] == null) { + return i; + } + } + } + else { + for (int i = startIndex; i < array.length; i++) { + if (objectToFind.equals(array[i])) { + return i; + } + } + } + + return -1; + } + + + /** + * 在数组中查找一个元素序列。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + *

+ * 起始索引小于0则看作0，超出数组长度的起始索引则返回-1。 + *

+ * + * @param array + * 要扫描的数组 + * @param arrayToFind + * 要查找的元素序列 + * @param startIndex + * 起始索引 + * + * @return 该元素序列在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int indexOf(Object[] array, Object[] arrayToFind, int startIndex) { + if ((array == null) || (arrayToFind == null)) { + return -1; + } + + int sourceLength = array.length; + int targetLength = arrayToFind.length; + + if (startIndex >= sourceLength) { + return (targetLength == 0) ? sourceLength : (-1); + } + + if (startIndex < 0) { + startIndex = 0; + } + + if (targetLength == 0) { + return startIndex; + } + + Object first = arrayToFind[0]; + int i = startIndex; + int max = sourceLength - targetLength; + + startSearchForFirst: while (true) { + // 查找第一个元素 + while ((i <= max) && !ObjectUtil.equals(array[i], first)) { + i++; + } + + if (i > max) { + return -1; + } + + // 已经找到第一个元素，接着找 + int j = i + 1; + int end = (j + targetLength) - 1; + int k = 1; + + while (j < end) { + if (!ObjectUtil.equals(array[j++], arrayToFind[k++])) { + i++; + + // 重新查找第一个元素 + continue startSearchForFirst; + } + } + + // 找到了 + return i; + } + } + + + /** + * 在数组中从末尾开始查找一个元素。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + * @param array + * 要扫描的数组 + * @param objectToFind + * 要查找的元素 + * + * @return 该元素在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int lastIndexOf(Object[] array, Object objectToFind) { + return lastIndexOf(array, objectToFind, Integer.MAX_VALUE); + } + + + /** + * 在数组中从末尾开始查找一个元素序列。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + * @param array + * 要扫描的数组 + * @param arrayToFind + * 要查找的元素序列 + * + * @return 该元素序列在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int lastIndexOf(Object[] array, Object[] arrayToFind) { + return lastIndexOf(array, arrayToFind, Integer.MAX_VALUE); + } + + + /** + * 在数组中从末尾开始查找一个元素。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + *

+ * 起始索引小于0则返回-1，超出数组长度的起始索引则从数组末尾开始找。 + *

+ * + * @param array + * 要扫描的数组 + * @param objectToFind + * 要查找的元素 + * @param startIndex + * 起始索引 + * + * @return 该元素在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int lastIndexOf(Object[] array, Object objectToFind, int startIndex) { + if (array == null) { + return -1; + } + + if (startIndex < 0) { + return -1; + } + else if (startIndex >= array.length) { + startIndex = array.length - 1; + } + + if (objectToFind == null) { + for (int i = startIndex; i >= 0; i--) { + if (array[i] == null) { + return i; + } + } + } + else { + for (int i = startIndex; i >= 0; i--) { + if (objectToFind.equals(array[i])) { + return i; + } + } + } + + return -1; + } + + + /** + * 在数组中从末尾开始查找一个元素序列。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + *

+ * 起始索引小于0则返回-1，超出数组长度的起始索引则从数组末尾开始找。 + *

+ * + * @param array + * 要扫描的数组 + * @param arrayToFind + * 要查找的元素序列 + * @param startIndex + * 起始索引 + * + * @return 该元素序列在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int lastIndexOf(Object[] array, Object[] arrayToFind, int startIndex) { + if ((array == null) || (arrayToFind == null)) { + return -1; + } + + int sourceLength = array.length; + int targetLength = arrayToFind.length; + + int rightIndex = sourceLength - targetLength; + + if (startIndex < 0) { + return -1; + } + + if (startIndex > rightIndex) { + startIndex = rightIndex; + } + + if (targetLength == 0) { + return startIndex; + } + + int lastIndex = targetLength - 1; + Object last = arrayToFind[lastIndex]; + int min = targetLength - 1; + int i = min + startIndex; + + startSearchForLast: while (true) { + while ((i >= min) && !ObjectUtil.equals(array[i], last)) { + i--; + } + + if (i < min) { + return -1; + } + + int j = i - 1; + int start = j - (targetLength - 1); + int k = lastIndex - 1; + + while (j > start) { + if (!ObjectUtil.equals(array[j--], arrayToFind[k--])) { + i--; + continue startSearchForLast; + } + } + + return start + 1; + } + } + + + /** + * 判断指定对象是否存在于指定数组中。 + * + *

+ * 如果数组为null则返回false。 + *

+ * + * @param array + * 要扫描的数组 + * @param objectToFind + * 要查找的元素 + * + * @return 如果找到则返回true + */ + public static boolean contains(Object[] array, Object objectToFind) { + return indexOf(array, objectToFind) != -1; + } + + + /** + * 判断指定元素序列是否存在于指定数组中。 + * + *

+ * 如果数组为null则返回false。 + *

+ * + * @param array + * 要扫描的数组 + * @param arrayToFind + * 要查找的元素序列 + * + * @return 如果找到则返回true + */ + public static boolean contains(Object[] array, Object[] arrayToFind) { + return indexOf(array, arrayToFind) != -1; + } + + + /* + * ========================================================================== + * == + */ + /* 在数组中查找一个元素或一个元素序列。 */ + /* */ + /* 类型：long[] */ + /* + * ========================================================================== + * == + */ + + /** + * 在数组中查找一个元素。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + * @param array + * 要扫描的数组 + * @param longToFind + * 要查找的元素 + * + * @return 该元素在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int indexOf(long[] array, long longToFind) { + return indexOf(array, longToFind, 0); + } + + + /** + * 在数组中查找一个元素序列。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + * @param array + * 要扫描的数组 + * @param arrayToFind + * 要查找的元素序列 + * + * @return 该元素序列在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int indexOf(long[] array, long[] arrayToFind) { + return indexOf(array, arrayToFind, 0); + } + + + /** + * 在数组中查找一个元素。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + *

+ * 起始索引小于0则看作0，超出数组长度的起始索引则返回-1。 + *

+ * + * @param array + * 要扫描的数组 + * @param longToFind + * 要查找的元素 + * @param startIndex + * 起始索引 + * + * @return 该元素在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int indexOf(long[] array, long longToFind, int startIndex) { + if (array == null) { + return -1; + } + + if (startIndex < 0) { + startIndex = 0; + } + + for (int i = startIndex; i < array.length; i++) { + if (longToFind == array[i]) { + return i; + } + } + + return -1; + } + + + /** + * 在数组中查找一个元素序列。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + *

+ * 起始索引小于0则看作0，超出数组长度的起始索引则返回-1。 + *

+ * + * @param array + * 要扫描的数组 + * @param arrayToFind + * 要查找的元素序列 + * @param startIndex + * 起始索引 + * + * @return 该元素序列在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int indexOf(long[] array, long[] arrayToFind, int startIndex) { + if ((array == null) || (arrayToFind == null)) { + return -1; + } + + int sourceLength = array.length; + int targetLength = arrayToFind.length; + + if (startIndex >= sourceLength) { + return (targetLength == 0) ? sourceLength : (-1); + } + + if (startIndex < 0) { + startIndex = 0; + } + + if (targetLength == 0) { + return startIndex; + } + + long first = arrayToFind[0]; + int i = startIndex; + int max = sourceLength - targetLength; + + startSearchForFirst: while (true) { + // 查找第一个元素 + while ((i <= max) && (array[i] != first)) { + i++; + } + + if (i > max) { + return -1; + } + + // 已经找到第一个元素，接着找 + int j = i + 1; + int end = (j + targetLength) - 1; + int k = 1; + + while (j < end) { + if (array[j++] != arrayToFind[k++]) { + i++; + + // 重新查找第一个元素 + continue startSearchForFirst; + } + } + + // 找到了 + return i; + } + } + + + /** + * 在数组中从末尾开始查找一个元素。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + * @param array + * 要扫描的数组 + * @param longToFind + * 要查找的元素 + * + * @return 该元素在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int lastIndexOf(long[] array, long longToFind) { + return lastIndexOf(array, longToFind, Integer.MAX_VALUE); + } + + + /** + * 在数组中从末尾开始查找一个元素序列。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + * @param array + * 要扫描的数组 + * @param arrayToFind + * 要查找的元素序列 + * + * @return 该元素序列在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int lastIndexOf(long[] array, long[] arrayToFind) { + return lastIndexOf(array, arrayToFind, Integer.MAX_VALUE); + } + + + /** + * 在数组中从末尾开始查找一个元素。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + *

+ * 起始索引小于0则返回-1，超出数组长度的起始索引则从数组末尾开始找。 + *

+ * + * @param array + * 要扫描的数组 + * @param longToFind + * 要查找的元素 + * @param startIndex + * 起始索引 + * + * @return 该元素在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int lastIndexOf(long[] array, long longToFind, int startIndex) { + if (array == null) { + return -1; + } + + if (startIndex < 0) { + return -1; + } + else if (startIndex >= array.length) { + startIndex = array.length - 1; + } + + for (int i = startIndex; i >= 0; i--) { + if (longToFind == array[i]) { + return i; + } + } + + return -1; + } + + + /** + * 在数组中从末尾开始查找一个元素序列。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + *

+ * 起始索引小于0则返回-1，超出数组长度的起始索引则从数组末尾开始找。 + *

+ * + * @param array + * 要扫描的数组 + * @param arrayToFind + * 要查找的元素序列 + * @param startIndex + * 起始索引 + * + * @return 该元素序列在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int lastIndexOf(long[] array, long[] arrayToFind, int startIndex) { + if ((array == null) || (arrayToFind == null)) { + return -1; + } + + int sourceLength = array.length; + int targetLength = arrayToFind.length; + + int rightIndex = sourceLength - targetLength; + + if (startIndex < 0) { + return -1; + } + + if (startIndex > rightIndex) { + startIndex = rightIndex; + } + + if (targetLength == 0) { + return startIndex; + } + + int lastIndex = targetLength - 1; + long last = arrayToFind[lastIndex]; + int min = targetLength - 1; + int i = min + startIndex; + + startSearchForLast: while (true) { + while ((i >= min) && (array[i] != last)) { + i--; + } + + if (i < min) { + return -1; + } + + int j = i - 1; + int start = j - (targetLength - 1); + int k = lastIndex - 1; + + while (j > start) { + if (array[j--] != arrayToFind[k--]) { + i--; + continue startSearchForLast; + } + } + + return start + 1; + } + } + + + /** + * 判断指定对象是否存在于指定数组中。 + * + *

+ * 如果数组为null则返回false。 + *

+ * + * @param array + * 要扫描的数组 + * @param longToFind + * 要查找的元素 + * + * @return 如果找到则返回true + */ + public static boolean contains(long[] array, long longToFind) { + return indexOf(array, longToFind) != -1; + } + + + /** + * 判断指定元素序列是否存在于指定数组中。 + * + *

+ * 如果数组为null则返回false。 + *

+ * + * @param array + * 要扫描的数组 + * @param arrayToFind + * 要查找的元素序列 + * + * @return 如果找到则返回true + */ + public static boolean contains(long[] array, long[] arrayToFind) { + return indexOf(array, arrayToFind) != -1; + } + + + /* + * ========================================================================== + * == + */ + /* 在数组中查找一个元素或一个元素序列。 */ + /* */ + /* 类型：int[] */ + /* + * ========================================================================== + * == + */ + + /** + * 在数组中查找一个元素。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + * @param array + * 要扫描的数组 + * @param intToFind + * 要查找的元素 + * + * @return 该元素在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int indexOf(int[] array, int intToFind) { + return indexOf(array, intToFind, 0); + } + + + /** + * 在数组中查找一个元素序列。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + * @param array + * 要扫描的数组 + * @param arrayToFind + * 要查找的元素序列 + * + * @return 该元素序列在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int indexOf(int[] array, int[] arrayToFind) { + return indexOf(array, arrayToFind, 0); + } + + + /** + * 在数组中查找一个元素。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + *

+ * 起始索引小于0则看作0，超出数组长度的起始索引则返回-1。 + *

+ * + * @param array + * 要扫描的数组 + * @param intToFind + * 要查找的元素 + * @param startIndex + * 起始索引 + * + * @return 该元素在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int indexOf(int[] array, int intToFind, int startIndex) { + if (array == null) { + return -1; + } + + if (startIndex < 0) { + startIndex = 0; + } + + for (int i = startIndex; i < array.length; i++) { + if (intToFind == array[i]) { + return i; + } + } + + return -1; + } + + + /** + * 在数组中查找一个元素序列。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + *

+ * 起始索引小于0则看作0，超出数组长度的起始索引则返回-1。 + *

+ * + * @param array + * 要扫描的数组 + * @param arrayToFind + * 要查找的元素序列 + * @param startIndex + * 起始索引 + * + * @return 该元素序列在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int indexOf(int[] array, int[] arrayToFind, int startIndex) { + if ((array == null) || (arrayToFind == null)) { + return -1; + } + + int sourceLength = array.length; + int targetLength = arrayToFind.length; + + if (startIndex >= sourceLength) { + return (targetLength == 0) ? sourceLength : (-1); + } + + if (startIndex < 0) { + startIndex = 0; + } + + if (targetLength == 0) { + return startIndex; + } + + int first = arrayToFind[0]; + int i = startIndex; + int max = sourceLength - targetLength; + + startSearchForFirst: while (true) { + // 查找第一个元素 + while ((i <= max) && (array[i] != first)) { + i++; + } + + if (i > max) { + return -1; + } + + // 已经找到第一个元素，接着找 + int j = i + 1; + int end = (j + targetLength) - 1; + int k = 1; + + while (j < end) { + if (array[j++] != arrayToFind[k++]) { + i++; + + // 重新查找第一个元素 + continue startSearchForFirst; + } + } + + // 找到了 + return i; + } + } + + + /** + * 在数组中从末尾开始查找一个元素。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + * @param array + * 要扫描的数组 + * @param intToFind + * 要查找的元素 + * + * @return 该元素在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int lastIndexOf(int[] array, int intToFind) { + return lastIndexOf(array, intToFind, Integer.MAX_VALUE); + } + + + /** + * 在数组中从末尾开始查找一个元素序列。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + * @param array + * 要扫描的数组 + * @param arrayToFind + * 要查找的元素序列 + * + * @return 该元素序列在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int lastIndexOf(int[] array, int[] arrayToFind) { + return lastIndexOf(array, arrayToFind, Integer.MAX_VALUE); + } + + + /** + * 在数组中从末尾开始查找一个元素。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + *

+ * 起始索引小于0则返回-1，超出数组长度的起始索引则从数组末尾开始找。 + *

+ * + * @param array + * 要扫描的数组 + * @param intToFind + * 要查找的元素 + * @param startIndex + * 起始索引 + * + * @return 该元素在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int lastIndexOf(int[] array, int intToFind, int startIndex) { + if (array == null) { + return -1; + } + + if (startIndex < 0) { + return -1; + } + else if (startIndex >= array.length) { + startIndex = array.length - 1; + } + + for (int i = startIndex; i >= 0; i--) { + if (intToFind == array[i]) { + return i; + } + } + + return -1; + } + + + /** + * 在数组中从末尾开始查找一个元素序列。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + *

+ * 起始索引小于0则返回-1，超出数组长度的起始索引则从数组末尾开始找。 + *

+ * + * @param array + * 要扫描的数组 + * @param arrayToFind + * 要查找的元素序列 + * @param startIndex + * 起始索引 + * + * @return 该元素序列在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int lastIndexOf(int[] array, int[] arrayToFind, int startIndex) { + if ((array == null) || (arrayToFind == null)) { + return -1; + } + + int sourceLength = array.length; + int targetLength = arrayToFind.length; + + int rightIndex = sourceLength - targetLength; + + if (startIndex < 0) { + return -1; + } + + if (startIndex > rightIndex) { + startIndex = rightIndex; + } + + if (targetLength == 0) { + return startIndex; + } + + int lastIndex = targetLength - 1; + int last = arrayToFind[lastIndex]; + int min = targetLength - 1; + int i = min + startIndex; + + startSearchForLast: while (true) { + while ((i >= min) && (array[i] != last)) { + i--; + } + + if (i < min) { + return -1; + } + + int j = i - 1; + int start = j - (targetLength - 1); + int k = lastIndex - 1; + + while (j > start) { + if (array[j--] != arrayToFind[k--]) { + i--; + continue startSearchForLast; + } + } + + return start + 1; + } + } + + + /** + * 判断指定对象是否存在于指定数组中。 + * + *

+ * 如果数组为null则返回false。 + *

+ * + * @param array + * 要扫描的数组 + * @param intToFind + * 要查找的元素 + * + * @return 如果找到则返回true + */ + public static boolean contains(int[] array, int intToFind) { + return indexOf(array, intToFind) != -1; + } + + + /** + * 判断指定元素序列是否存在于指定数组中。 + * + *

+ * 如果数组为null则返回false。 + *

+ * + * @param array + * 要扫描的数组 + * @param arrayToFind + * 要查找的元素序列 + * + * @return 如果找到则返回true + */ + public static boolean contains(int[] array, int[] arrayToFind) { + return indexOf(array, arrayToFind) != -1; + } + + + /* + * ========================================================================== + * == + */ + /* 在数组中查找一个元素或一个元素序列。 */ + /* */ + /* 类型：short[] */ + /* + * ========================================================================== + * == + */ + + /** + * 在数组中查找一个元素。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + * @param array + * 要扫描的数组 + * @param shortToFind + * 要查找的元素 + * + * @return 该元素在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int indexOf(short[] array, short shortToFind) { + return indexOf(array, shortToFind, 0); + } + + + /** + * 在数组中查找一个元素序列。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + * @param array + * 要扫描的数组 + * @param arrayToFind + * 要查找的元素序列 + * + * @return 该元素序列在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int indexOf(short[] array, short[] arrayToFind) { + return indexOf(array, arrayToFind, 0); + } + + + /** + * 在数组中查找一个元素。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + *

+ * 起始索引小于0则看作0，超出数组长度的起始索引则返回-1。 + *

+ * + * @param array + * 要扫描的数组 + * @param shortToFind + * 要查找的元素 + * @param startIndex + * 起始索引 + * + * @return 该元素在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int indexOf(short[] array, short shortToFind, int startIndex) { + if (array == null) { + return -1; + } + + if (startIndex < 0) { + startIndex = 0; + } + + for (int i = startIndex; i < array.length; i++) { + if (shortToFind == array[i]) { + return i; + } + } + + return -1; + } + + + /** + * 在数组中查找一个元素序列。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + *

+ * 起始索引小于0则看作0，超出数组长度的起始索引则返回-1。 + *

+ * + * @param array + * 要扫描的数组 + * @param arrayToFind + * 要查找的元素序列 + * @param startIndex + * 起始索引 + * + * @return 该元素序列在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int indexOf(short[] array, short[] arrayToFind, int startIndex) { + if ((array == null) || (arrayToFind == null)) { + return -1; + } + + int sourceLength = array.length; + int targetLength = arrayToFind.length; + + if (startIndex >= sourceLength) { + return (targetLength == 0) ? sourceLength : (-1); + } + + if (startIndex < 0) { + startIndex = 0; + } + + if (targetLength == 0) { + return startIndex; + } + + short first = arrayToFind[0]; + int i = startIndex; + int max = sourceLength - targetLength; + + startSearchForFirst: while (true) { + // 查找第一个元素 + while ((i <= max) && (array[i] != first)) { + i++; + } + + if (i > max) { + return -1; + } + + // 已经找到第一个元素，接着找 + int j = i + 1; + int end = (j + targetLength) - 1; + int k = 1; + + while (j < end) { + if (array[j++] != arrayToFind[k++]) { + i++; + + // 重新查找第一个元素 + continue startSearchForFirst; + } + } + + // 找到了 + return i; + } + } + + + /** + * 在数组中从末尾开始查找一个元素。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + * @param array + * 要扫描的数组 + * @param shortToFind + * 要查找的元素 + * + * @return 该元素在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int lastIndexOf(short[] array, short shortToFind) { + return lastIndexOf(array, shortToFind, Integer.MAX_VALUE); + } + + + /** + * 在数组中从末尾开始查找一个元素序列。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + * @param array + * 要扫描的数组 + * @param arrayToFind + * 要查找的元素序列 + * + * @return 该元素序列在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int lastIndexOf(short[] array, short[] arrayToFind) { + return lastIndexOf(array, arrayToFind, Integer.MAX_VALUE); + } + + + /** + * 在数组中从末尾开始查找一个元素。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + *

+ * 起始索引小于0则返回-1，超出数组长度的起始索引则从数组末尾开始找。 + *

+ * + * @param array + * 要扫描的数组 + * @param shortToFind + * 要查找的元素 + * @param startIndex + * 起始索引 + * + * @return 该元素在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int lastIndexOf(short[] array, short shortToFind, int startIndex) { + if (array == null) { + return -1; + } + + if (startIndex < 0) { + return -1; + } + else if (startIndex >= array.length) { + startIndex = array.length - 1; + } + + for (int i = startIndex; i >= 0; i--) { + if (shortToFind == array[i]) { + return i; + } + } + + return -1; + } + + + /** + * 在数组中从末尾开始查找一个元素序列。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + *

+ * 起始索引小于0则返回-1，超出数组长度的起始索引则从数组末尾开始找。 + *

+ * + * @param array + * 要扫描的数组 + * @param arrayToFind + * 要查找的元素序列 + * @param startIndex + * 起始索引 + * + * @return 该元素序列在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int lastIndexOf(short[] array, short[] arrayToFind, int startIndex) { + if ((array == null) || (arrayToFind == null)) { + return -1; + } + + int sourceLength = array.length; + int targetLength = arrayToFind.length; + + int rightIndex = sourceLength - targetLength; + + if (startIndex < 0) { + return -1; + } + + if (startIndex > rightIndex) { + startIndex = rightIndex; + } + + if (targetLength == 0) { + return startIndex; + } + + int lastIndex = targetLength - 1; + short last = arrayToFind[lastIndex]; + int min = targetLength - 1; + int i = min + startIndex; + + startSearchForLast: while (true) { + while ((i >= min) && (array[i] != last)) { + i--; + } + + if (i < min) { + return -1; + } + + int j = i - 1; + int start = j - (targetLength - 1); + int k = lastIndex - 1; + + while (j > start) { + if (array[j--] != arrayToFind[k--]) { + i--; + continue startSearchForLast; + } + } + + return start + 1; + } + } + + + /** + * 判断指定对象是否存在于指定数组中。 + * + *

+ * 如果数组为null则返回false。 + *

+ * + * @param array + * 要扫描的数组 + * @param shortToFind + * 要查找的元素 + * + * @return 如果找到则返回true + */ + public static boolean contains(short[] array, short shortToFind) { + return indexOf(array, shortToFind) != -1; + } + + + /** + * 判断指定元素序列是否存在于指定数组中。 + * + *

+ * 如果数组为null则返回false。 + *

+ * + * @param array + * 要扫描的数组 + * @param arrayToFind + * 要查找的元素序列 + * + * @return 如果找到则返回true + */ + public static boolean contains(short[] array, short[] arrayToFind) { + return indexOf(array, arrayToFind) != -1; + } + + + /* + * ========================================================================== + * == + */ + /* 在数组中查找一个元素或一个元素序列。 */ + /* */ + /* 类型：byte[] */ + /* + * ========================================================================== + * == + */ + + /** + * 在数组中查找一个元素。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + * @param array + * 要扫描的数组 + * @param byteToFind + * 要查找的元素 + * + * @return 该元素在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int indexOf(byte[] array, byte byteToFind) { + return indexOf(array, byteToFind, 0); + } + + + /** + * 在数组中查找一个元素序列。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + * @param array + * 要扫描的数组 + * @param arrayToFind + * 要查找的元素序列 + * + * @return 该元素序列在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int indexOf(byte[] array, byte[] arrayToFind) { + return indexOf(array, arrayToFind, 0); + } + + + /** + * 在数组中查找一个元素。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + *

+ * 起始索引小于0则看作0，超出数组长度的起始索引则返回-1。 + *

+ * + * @param array + * 要扫描的数组 + * @param byteToFind + * 要查找的元素 + * @param startIndex + * 起始索引 + * + * @return 该元素在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int indexOf(byte[] array, byte byteToFind, int startIndex) { + if (array == null) { + return -1; + } + + if (startIndex < 0) { + startIndex = 0; + } + + for (int i = startIndex; i < array.length; i++) { + if (byteToFind == array[i]) { + return i; + } + } + + return -1; + } + + + /** + * 在数组中查找一个元素序列。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + *

+ * 起始索引小于0则看作0，超出数组长度的起始索引则返回-1。 + *

+ * + * @param array + * 要扫描的数组 + * @param arrayToFind + * 要查找的元素序列 + * @param startIndex + * 起始索引 + * + * @return 该元素序列在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int indexOf(byte[] array, byte[] arrayToFind, int startIndex) { + if ((array == null) || (arrayToFind == null)) { + return -1; + } + + int sourceLength = array.length; + int targetLength = arrayToFind.length; + + if (startIndex >= sourceLength) { + return (targetLength == 0) ? sourceLength : (-1); + } + + if (startIndex < 0) { + startIndex = 0; + } + + if (targetLength == 0) { + return startIndex; + } + + byte first = arrayToFind[0]; + int i = startIndex; + int max = sourceLength - targetLength; + + startSearchForFirst: while (true) { + // 查找第一个元素 + while ((i <= max) && (array[i] != first)) { + i++; + } + + if (i > max) { + return -1; + } + + // 已经找到第一个元素，接着找 + int j = i + 1; + int end = (j + targetLength) - 1; + int k = 1; + + while (j < end) { + if (array[j++] != arrayToFind[k++]) { + i++; + + // 重新查找第一个元素 + continue startSearchForFirst; + } + } + + // 找到了 + return i; + } + } + + + /** + * 在数组中从末尾开始查找一个元素。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + * @param array + * 要扫描的数组 + * @param byteToFind + * 要查找的元素 + * + * @return 该元素在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int lastIndexOf(byte[] array, byte byteToFind) { + return lastIndexOf(array, byteToFind, Integer.MAX_VALUE); + } + + + /** + * 在数组中从末尾开始查找一个元素序列。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + * @param array + * 要扫描的数组 + * @param arrayToFind + * 要查找的元素序列 + * + * @return 该元素序列在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int lastIndexOf(byte[] array, byte[] arrayToFind) { + return lastIndexOf(array, arrayToFind, Integer.MAX_VALUE); + } + + + /** + * 在数组中从末尾开始查找一个元素。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + *

+ * 起始索引小于0则返回-1，超出数组长度的起始索引则从数组末尾开始找。 + *

+ * + * @param array + * 要扫描的数组 + * @param byteToFind + * 要查找的元素 + * @param startIndex + * 起始索引 + * + * @return 该元素在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int lastIndexOf(byte[] array, byte byteToFind, int startIndex) { + if (array == null) { + return -1; + } + + if (startIndex < 0) { + return -1; + } + else if (startIndex >= array.length) { + startIndex = array.length - 1; + } + + for (int i = startIndex; i >= 0; i--) { + if (byteToFind == array[i]) { + return i; + } + } + + return -1; + } + + + /** + * 在数组中从末尾开始查找一个元素序列。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + *

+ * 起始索引小于0则返回-1，超出数组长度的起始索引则从数组末尾开始找。 + *

+ * + * @param array + * 要扫描的数组 + * @param arrayToFind + * 要查找的元素序列 + * @param startIndex + * 起始索引 + * + * @return 该元素序列在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int lastIndexOf(byte[] array, byte[] arrayToFind, int startIndex) { + if ((array == null) || (arrayToFind == null)) { + return -1; + } + + int sourceLength = array.length; + int targetLength = arrayToFind.length; + + int rightIndex = sourceLength - targetLength; + + if (startIndex < 0) { + return -1; + } + + if (startIndex > rightIndex) { + startIndex = rightIndex; + } + + if (targetLength == 0) { + return startIndex; + } + + int lastIndex = targetLength - 1; + byte last = arrayToFind[lastIndex]; + int min = targetLength - 1; + int i = min + startIndex; + + startSearchForLast: while (true) { + while ((i >= min) && (array[i] != last)) { + i--; + } + + if (i < min) { + return -1; + } + + int j = i - 1; + int start = j - (targetLength - 1); + int k = lastIndex - 1; + + while (j > start) { + if (array[j--] != arrayToFind[k--]) { + i--; + continue startSearchForLast; + } + } + + return start + 1; + } + } + + + /** + * 判断指定对象是否存在于指定数组中。 + * + *

+ * 如果数组为null则返回false。 + *

+ * + * @param array + * 要扫描的数组 + * @param byteToFind + * 要查找的元素 + * + * @return 如果找到则返回true + */ + public static boolean contains(byte[] array, byte byteToFind) { + return indexOf(array, byteToFind) != -1; + } + + + /** + * 判断指定元素序列是否存在于指定数组中。 + * + *

+ * 如果数组为null则返回false。 + *

+ * + * @param array + * 要扫描的数组 + * @param arrayToFind + * 要查找的元素序列 + * + * @return 如果找到则返回true + */ + public static boolean contains(byte[] array, byte[] arrayToFind) { + return indexOf(array, arrayToFind) != -1; + } + + + /* + * ========================================================================== + * == + */ + /* 在数组中查找一个元素或一个元素序列。 */ + /* */ + /* 类型：double[] */ + /* + * ========================================================================== + * == + */ + + /** + * 在数组中查找一个元素。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + * @param array + * 要扫描的数组 + * @param doubleToFind + * 要查找的元素 + * + * @return 该元素在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int indexOf(double[] array, double doubleToFind) { + return indexOf(array, doubleToFind, 0, 0); + } + + + /** + * 在数组中查找一个元素。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + * @param array + * 要扫描的数组 + * @param doubleToFind + * 要查找的元素 + * @param tolerance + * 误差 + * + * @return 该元素在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int indexOf(double[] array, double doubleToFind, double tolerance) { + return indexOf(array, doubleToFind, 0, tolerance); + } + + + /** + * 在数组中查找一个元素序列。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + * @param array + * 要扫描的数组 + * @param arrayToFind + * 要查找的元素序列 + * + * @return 该元素序列在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int indexOf(double[] array, double[] arrayToFind) { + return indexOf(array, arrayToFind, 0, 0); + } + + + /** + * 在数组中查找一个元素序列。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + * @param array + * 要扫描的数组 + * @param arrayToFind + * 要查找的元素序列 + * @param tolerance + * 误差 + * + * @return 该元素序列在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int indexOf(double[] array, double[] arrayToFind, double tolerance) { + return indexOf(array, arrayToFind, 0, tolerance); + } + + + /** + * 在数组中查找一个元素。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + *

+ * 起始索引小于0则看作0，超出数组长度的起始索引则返回-1。 + *

+ * + * @param array + * 要扫描的数组 + * @param doubleToFind + * 要查找的元素 + * @param startIndex + * 起始索引 + * + * @return 该元素在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int indexOf(double[] array, double doubleToFind, int startIndex) { + return indexOf(array, doubleToFind, startIndex, 0); + } + + + /** + * 在数组中查找一个元素。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + *

+ * 起始索引小于0则看作0，超出数组长度的起始索引则返回-1。 + *

+ * + * @param array + * 要扫描的数组 + * @param doubleToFind + * 要查找的元素 + * @param startIndex + * 起始索引 + * @param tolerance + * 误差 + * + * @return 该元素在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int indexOf(double[] array, double doubleToFind, int startIndex, double tolerance) { + if (array == null) { + return -1; + } + + if (startIndex < 0) { + startIndex = 0; + } + + double min = doubleToFind - tolerance; + double max = doubleToFind + tolerance; + + for (int i = startIndex; i < array.length; i++) { + if ((array[i] >= min) && (array[i] <= max)) { + return i; + } + } + + return -1; + } + + + /** + * 在数组中查找一个元素序列。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + *

+ * 起始索引小于0则看作0，超出数组长度的起始索引则返回-1。 + *

+ * + * @param array + * 要扫描的数组 + * @param arrayToFind + * 要查找的元素序列 + * @param startIndex + * 起始索引 + * + * @return 该元素序列在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int indexOf(double[] array, double[] arrayToFind, int startIndex) { + return indexOf(array, arrayToFind, startIndex, 0); + } + + + /** + * 在数组中查找一个元素序列。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + *

+ * 起始索引小于0则看作0，超出数组长度的起始索引则返回-1。 + *

+ * + * @param array + * 要扫描的数组 + * @param arrayToFind + * 要查找的元素序列 + * @param startIndex + * 起始索引 + * @param tolerance + * 误差 + * + * @return 该元素序列在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int indexOf(double[] array, double[] arrayToFind, int startIndex, double tolerance) { + if ((array == null) || (arrayToFind == null)) { + return -1; + } + + int sourceLength = array.length; + int targetLength = arrayToFind.length; + + if (startIndex >= sourceLength) { + return (targetLength == 0) ? sourceLength : (-1); + } + + if (startIndex < 0) { + startIndex = 0; + } + + if (targetLength == 0) { + return startIndex; + } + + double firstMin = arrayToFind[0] - tolerance; + double firstMax = arrayToFind[0] + tolerance; + int i = startIndex; + int max = sourceLength - targetLength; + + startSearchForFirst: while (true) { + // 查找第一个元素 + while ((i <= max) && ((array[i] < firstMin) || (array[i] > firstMax))) { + i++; + } + + if (i > max) { + return -1; + } + + // 已经找到第一个元素，接着找 + int j = i + 1; + int end = (j + targetLength) - 1; + int k = 1; + + while (j < end) { + if (Math.abs(array[j++] - arrayToFind[k++]) > tolerance) { + i++; + + // 重新查找第一个元素 + continue startSearchForFirst; + } + } + + // 找到了 + return i; + } + } + + + /** + * 在数组中从末尾开始查找一个元素。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + * @param array + * 要扫描的数组 + * @param doubleToFind + * 要查找的元素 + * + * @return 该元素在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int lastIndexOf(double[] array, double doubleToFind) { + return lastIndexOf(array, doubleToFind, Integer.MAX_VALUE, 0); + } + + + /** + * 在数组中从末尾开始查找一个元素。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + * @param array + * 要扫描的数组 + * @param doubleToFind + * 要查找的元素 + * @param tolerance + * 误差 + * + * @return 该元素在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int lastIndexOf(double[] array, double doubleToFind, double tolerance) { + return lastIndexOf(array, doubleToFind, Integer.MAX_VALUE, tolerance); + } + + + /** + * 在数组中从末尾开始查找一个元素序列。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + * @param array + * 要扫描的数组 + * @param arrayToFind + * 要查找的元素序列 + * + * @return 该元素序列在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int lastIndexOf(double[] array, double[] arrayToFind) { + return lastIndexOf(array, arrayToFind, Integer.MAX_VALUE, 0); + } + + + /** + * 在数组中从末尾开始查找一个元素序列。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + * @param array + * 要扫描的数组 + * @param arrayToFind + * 要查找的元素序列 + * @param tolerance + * 误差 + * + * @return 该元素序列在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int lastIndexOf(double[] array, double[] arrayToFind, double tolerance) { + return lastIndexOf(array, arrayToFind, Integer.MAX_VALUE, tolerance); + } + + + /** + * 在数组中从末尾开始查找一个元素。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + *

+ * 起始索引小于0则返回-1，超出数组长度的起始索引则从数组末尾开始找。 + *

+ * + * @param array + * 要扫描的数组 + * @param doubleToFind + * 要查找的元素 + * @param startIndex + * 起始索引 + * + * @return 该元素在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int lastIndexOf(double[] array, double doubleToFind, int startIndex) { + return lastIndexOf(array, doubleToFind, startIndex, 0); + } + + + /** + * 在数组中从末尾开始查找一个元素。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + *

+ * 起始索引小于0则返回-1，超出数组长度的起始索引则从数组末尾开始找。 + *

+ * + * @param array + * 要扫描的数组 + * @param doubleToFind + * 要查找的元素 + * @param startIndex + * 起始索引 + * @param tolerance + * 误差 + * + * @return 该元素在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int lastIndexOf(double[] array, double doubleToFind, int startIndex, double tolerance) { + if (array == null) { + return -1; + } + + if (startIndex < 0) { + return -1; + } + else if (startIndex >= array.length) { + startIndex = array.length - 1; + } + + double min = doubleToFind - tolerance; + double max = doubleToFind + tolerance; + + for (int i = startIndex; i >= 0; i--) { + if ((array[i] >= min) && (array[i] <= max)) { + return i; + } + } + + return -1; + } + + + /** + * 在数组中从末尾开始查找一个元素序列。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + *

+ * 起始索引小于0则返回-1，超出数组长度的起始索引则从数组末尾开始找。 + *

+ * + * @param array + * 要扫描的数组 + * @param arrayToFind + * 要查找的元素序列 + * @param startIndex + * 起始索引 + * + * @return 该元素序列在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int lastIndexOf(double[] array, double[] arrayToFind, int startIndex) { + return lastIndexOf(array, arrayToFind, startIndex, 0); + } + + + /** + * 在数组中从末尾开始查找一个元素序列。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + *

+ * 起始索引小于0则返回-1，超出数组长度的起始索引则从数组末尾开始找。 + *

+ * + * @param array + * 要扫描的数组 + * @param arrayToFind + * 要查找的元素序列 + * @param startIndex + * 起始索引 + * @param tolerance + * 误差 + * + * @return 该元素序列在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int lastIndexOf(double[] array, double[] arrayToFind, int startIndex, double tolerance) { + if ((array == null) || (arrayToFind == null)) { + return -1; + } + + int sourceLength = array.length; + int targetLength = arrayToFind.length; + + int rightIndex = sourceLength - targetLength; + + if (startIndex < 0) { + return -1; + } + + if (startIndex > rightIndex) { + startIndex = rightIndex; + } + + if (targetLength == 0) { + return startIndex; + } + + int lastIndex = targetLength - 1; + double lastMin = arrayToFind[lastIndex] - tolerance; + double lastMax = arrayToFind[lastIndex] + tolerance; + int min = targetLength - 1; + int i = min + startIndex; + + startSearchForLast: while (true) { + while ((i >= min) && ((array[i] < lastMin) || (array[i] > lastMax))) { + i--; + } + + if (i < min) { + return -1; + } + + int j = i - 1; + int start = j - (targetLength - 1); + int k = lastIndex - 1; + + while (j > start) { + if (Math.abs(array[j--] - arrayToFind[k--]) > tolerance) { + i--; + continue startSearchForLast; + } + } + + return start + 1; + } + } + + + /** + * 判断指定对象是否存在于指定数组中。 + * + *

+ * 如果数组为null则返回false。 + *

+ * + * @param array + * 要扫描的数组 + * @param doubleToFind + * 要查找的元素 + * + * @return 如果找到则返回true + */ + public static boolean contains(double[] array, double doubleToFind) { + return indexOf(array, doubleToFind) != -1; + } + + + /** + * 判断指定对象是否存在于指定数组中。 + * + *

+ * 如果数组为null则返回false。 + *

+ * + * @param array + * 要扫描的数组 + * @param doubleToFind + * 要查找的元素 + * @param tolerance + * 误差 + * + * @return 如果找到则返回true + */ + public static boolean contains(double[] array, double doubleToFind, double tolerance) { + return indexOf(array, doubleToFind, tolerance) != -1; + } + + + /** + * 判断指定元素序列是否存在于指定数组中。 + * + *

+ * 如果数组为null则返回false。 + *

+ * + * @param array + * 要扫描的数组 + * @param arrayToFind + * 要查找的元素序列 + * + * @return 如果找到则返回true + */ + public static boolean contains(double[] array, double[] arrayToFind) { + return indexOf(array, arrayToFind) != -1; + } + + + /** + * 判断指定元素序列是否存在于指定数组中。 + * + *

+ * 如果数组为null则返回false。 + *

+ * + * @param array + * 要扫描的数组 + * @param arrayToFind + * 要查找的元素序列 + * @param tolerance + * 误差 + * + * @return 如果找到则返回true + */ + public static boolean contains(double[] array, double[] arrayToFind, double tolerance) { + return indexOf(array, arrayToFind, tolerance) != -1; + } + + + /* + * ========================================================================== + * == + */ + /* 在数组中查找一个元素或一个元素序列。 */ + /* */ + /* 类型：float[] */ + /* + * ========================================================================== + * == + */ + + /** + * 在数组中查找一个元素。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + * @param array + * 要扫描的数组 + * @param floatToFind + * 要查找的元素 + * + * @return 该元素在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int indexOf(float[] array, float floatToFind) { + return indexOf(array, floatToFind, 0, 0); + } + + + /** + * 在数组中查找一个元素。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + * @param array + * 要扫描的数组 + * @param floatToFind + * 要查找的元素 + * @param tolerance + * 误差 + * + * @return 该元素在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int indexOf(float[] array, float floatToFind, float tolerance) { + return indexOf(array, floatToFind, 0, tolerance); + } + + + /** + * 在数组中查找一个元素序列。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + * @param array + * 要扫描的数组 + * @param arrayToFind + * 要查找的元素序列 + * + * @return 该元素序列在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int indexOf(float[] array, float[] arrayToFind) { + return indexOf(array, arrayToFind, 0, 0); + } + + + /** + * 在数组中查找一个元素序列。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + * @param array + * 要扫描的数组 + * @param arrayToFind + * 要查找的元素序列 + * @param tolerance + * 误差 + * + * @return 该元素序列在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int indexOf(float[] array, float[] arrayToFind, float tolerance) { + return indexOf(array, arrayToFind, 0, tolerance); + } + + + /** + * 在数组中查找一个元素。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + *

+ * 起始索引小于0则看作0，超出数组长度的起始索引则返回-1。 + *

+ * + * @param array + * 要扫描的数组 + * @param floatToFind + * 要查找的元素 + * @param startIndex + * 起始索引 + * + * @return 该元素在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int indexOf(float[] array, float floatToFind, int startIndex) { + return indexOf(array, floatToFind, startIndex, 0); + } + + + /** + * 在数组中查找一个元素。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + *

+ * 起始索引小于0则看作0，超出数组长度的起始索引则返回-1。 + *

+ * + * @param array + * 要扫描的数组 + * @param floatToFind + * 要查找的元素 + * @param startIndex + * 起始索引 + * @param tolerance + * 误差 + * + * @return 该元素在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int indexOf(float[] array, float floatToFind, int startIndex, float tolerance) { + if (array == null) { + return -1; + } + + if (startIndex < 0) { + startIndex = 0; + } + + float min = floatToFind - tolerance; + float max = floatToFind + tolerance; + + for (int i = startIndex; i < array.length; i++) { + if ((array[i] >= min) && (array[i] <= max)) { + return i; + } + } + + return -1; + } + + + /** + * 在数组中查找一个元素序列。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + *

+ * 起始索引小于0则看作0，超出数组长度的起始索引则返回-1。 + *

+ * + * @param array + * 要扫描的数组 + * @param arrayToFind + * 要查找的元素序列 + * @param startIndex + * 起始索引 + * + * @return 该元素序列在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int indexOf(float[] array, float[] arrayToFind, int startIndex) { + return indexOf(array, arrayToFind, startIndex, 0); + } + + + /** + * 在数组中查找一个元素序列。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + *

+ * 起始索引小于0则看作0，超出数组长度的起始索引则返回-1。 + *

+ * + * @param array + * 要扫描的数组 + * @param arrayToFind + * 要查找的元素序列 + * @param startIndex + * 起始索引 + * @param tolerance + * 误差 + * + * @return 该元素序列在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int indexOf(float[] array, float[] arrayToFind, int startIndex, float tolerance) { + if ((array == null) || (arrayToFind == null)) { + return -1; + } + + int sourceLength = array.length; + int targetLength = arrayToFind.length; + + if (startIndex >= sourceLength) { + return (targetLength == 0) ? sourceLength : (-1); + } + + if (startIndex < 0) { + startIndex = 0; + } + + if (targetLength == 0) { + return startIndex; + } + + float firstMin = arrayToFind[0] - tolerance; + float firstMax = arrayToFind[0] + tolerance; + int i = startIndex; + int max = sourceLength - targetLength; + + startSearchForFirst: while (true) { + // 查找第一个元素 + while ((i <= max) && ((array[i] < firstMin) || (array[i] > firstMax))) { + i++; + } + + if (i > max) { + return -1; + } + + // 已经找到第一个元素，接着找 + int j = i + 1; + int end = (j + targetLength) - 1; + int k = 1; + + while (j < end) { + if (Math.abs(array[j++] - arrayToFind[k++]) > tolerance) { + i++; + + // 重新查找第一个元素 + continue startSearchForFirst; + } + } + + // 找到了 + return i; + } + } + + + /** + * 在数组中从末尾开始查找一个元素。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + * @param array + * 要扫描的数组 + * @param floatToFind + * 要查找的元素 + * + * @return 该元素在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int lastIndexOf(float[] array, float floatToFind) { + return lastIndexOf(array, floatToFind, Integer.MAX_VALUE, 0); + } + + + /** + * 在数组中从末尾开始查找一个元素。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + * @param array + * 要扫描的数组 + * @param floatToFind + * 要查找的元素 + * @param tolerance + * 误差 + * + * @return 该元素在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int lastIndexOf(float[] array, float floatToFind, float tolerance) { + return lastIndexOf(array, floatToFind, Integer.MAX_VALUE, tolerance); + } + + + /** + * 在数组中从末尾开始查找一个元素序列。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + * @param array + * 要扫描的数组 + * @param arrayToFind + * 要查找的元素序列 + * + * @return 该元素序列在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int lastIndexOf(float[] array, float[] arrayToFind) { + return lastIndexOf(array, arrayToFind, Integer.MAX_VALUE, 0); + } + + + /** + * 在数组中从末尾开始查找一个元素序列。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + * @param array + * 要扫描的数组 + * @param arrayToFind + * 要查找的元素序列 + * @param tolerance + * 误差 + * + * @return 该元素序列在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int lastIndexOf(float[] array, float[] arrayToFind, float tolerance) { + return lastIndexOf(array, arrayToFind, Integer.MAX_VALUE, tolerance); + } + + + /** + * 在数组中从末尾开始查找一个元素。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + *

+ * 起始索引小于0则返回-1，超出数组长度的起始索引则从数组末尾开始找。 + *

+ * + * @param array + * 要扫描的数组 + * @param floatToFind + * 要查找的元素 + * @param startIndex + * 起始索引 + * + * @return 该元素在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int lastIndexOf(float[] array, float floatToFind, int startIndex) { + return lastIndexOf(array, floatToFind, startIndex, 0); + } + + + /** + * 在数组中从末尾开始查找一个元素。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + *

+ * 起始索引小于0则返回-1，超出数组长度的起始索引则从数组末尾开始找。 + *

+ * + * @param array + * 要扫描的数组 + * @param floatToFind + * 要查找的元素 + * @param startIndex + * 起始索引 + * @param tolerance + * 误差 + * + * @return 该元素在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int lastIndexOf(float[] array, float floatToFind, int startIndex, float tolerance) { + if (array == null) { + return -1; + } + + if (startIndex < 0) { + return -1; + } + else if (startIndex >= array.length) { + startIndex = array.length - 1; + } + + float min = floatToFind - tolerance; + float max = floatToFind + tolerance; + + for (int i = startIndex; i >= 0; i--) { + if ((array[i] >= min) && (array[i] <= max)) { + return i; + } + } + + return -1; + } + + + /** + * 在数组中从末尾开始查找一个元素序列。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + *

+ * 起始索引小于0则返回-1，超出数组长度的起始索引则从数组末尾开始找。 + *

+ * + * @param array + * 要扫描的数组 + * @param arrayToFind + * 要查找的元素序列 + * @param startIndex + * 起始索引 + * + * @return 该元素序列在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int lastIndexOf(float[] array, float[] arrayToFind, int startIndex) { + return lastIndexOf(array, arrayToFind, startIndex, 0); + } + + + /** + * 在数组中从末尾开始查找一个元素序列。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + *

+ * 起始索引小于0则返回-1，超出数组长度的起始索引则从数组末尾开始找。 + *

+ * + * @param array + * 要扫描的数组 + * @param arrayToFind + * 要查找的元素序列 + * @param startIndex + * 起始索引 + * @param tolerance + * 误差 + * + * @return 该元素序列在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int lastIndexOf(float[] array, float[] arrayToFind, int startIndex, float tolerance) { + if ((array == null) || (arrayToFind == null)) { + return -1; + } + + int sourceLength = array.length; + int targetLength = arrayToFind.length; + + int rightIndex = sourceLength - targetLength; + + if (startIndex < 0) { + return -1; + } + + if (startIndex > rightIndex) { + startIndex = rightIndex; + } + + if (targetLength == 0) { + return startIndex; + } + + int lastIndex = targetLength - 1; + float lastMin = arrayToFind[lastIndex] - tolerance; + float lastMax = arrayToFind[lastIndex] + tolerance; + int min = targetLength - 1; + int i = min + startIndex; + + startSearchForLast: while (true) { + while ((i >= min) && ((array[i] < lastMin) || (array[i] > lastMax))) { + i--; + } + + if (i < min) { + return -1; + } + + int j = i - 1; + int start = j - (targetLength - 1); + int k = lastIndex - 1; + + while (j > start) { + if (Math.abs(array[j--] - arrayToFind[k--]) > tolerance) { + i--; + continue startSearchForLast; + } + } + + return start + 1; + } + } + + + /** + * 判断指定对象是否存在于指定数组中。 + * + *

+ * 如果数组为null则返回false。 + *

+ * + * @param array + * 要扫描的数组 + * @param floatToFind + * 要查找的元素 + * + * @return 如果找到则返回true + */ + public static boolean contains(float[] array, float floatToFind) { + return indexOf(array, floatToFind) != -1; + } + + + /** + * 判断指定对象是否存在于指定数组中。 + * + *

+ * 如果数组为null则返回false。 + *

+ * + * @param array + * 要扫描的数组 + * @param floatToFind + * 要查找的元素 + * @param tolerance + * 误差 + * + * @return 如果找到则返回true + */ + public static boolean contains(float[] array, float floatToFind, float tolerance) { + return indexOf(array, floatToFind, tolerance) != -1; + } + + + /** + * 判断指定元素序列是否存在于指定数组中。 + * + *

+ * 如果数组为null则返回false。 + *

+ * + * @param array + * 要扫描的数组 + * @param arrayToFind + * 要查找的元素序列 + * + * @return 如果找到则返回true + */ + public static boolean contains(float[] array, float[] arrayToFind) { + return indexOf(array, arrayToFind) != -1; + } + + + /** + * 判断指定元素序列是否存在于指定数组中。 + * + *

+ * 如果数组为null则返回false。 + *

+ * + * @param array + * 要扫描的数组 + * @param arrayToFind + * 要查找的元素序列 + * @param tolerance + * 误差 + * + * @return 如果找到则返回true + */ + public static boolean contains(float[] array, float[] arrayToFind, float tolerance) { + return indexOf(array, arrayToFind, tolerance) != -1; + } + + + /* + * ========================================================================== + * == + */ + /* 在数组中查找一个元素或一个元素序列。 */ + /* */ + /* 类型：boolean[] */ + /* + * ========================================================================== + * == + */ + + /** + * 在数组中查找一个元素。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + * @param array + * 要扫描的数组 + * @param booleanToFind + * 要查找的元素 + * + * @return 该元素在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int indexOf(boolean[] array, boolean booleanToFind) { + return indexOf(array, booleanToFind, 0); + } + + + /** + * 在数组中查找一个元素序列。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + * @param array + * 要扫描的数组 + * @param arrayToFind + * 要查找的元素序列 + * + * @return 该元素序列在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int indexOf(boolean[] array, boolean[] arrayToFind) { + return indexOf(array, arrayToFind, 0); + } + + + /** + * 在数组中查找一个元素。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + *

+ * 起始索引小于0则看作0，超出数组长度的起始索引则返回-1。 + *

+ * + * @param array + * 要扫描的数组 + * @param booleanToFind + * 要查找的元素 + * @param startIndex + * 起始索引 + * + * @return 该元素在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int indexOf(boolean[] array, boolean booleanToFind, int startIndex) { + if (array == null) { + return -1; + } + + if (startIndex < 0) { + startIndex = 0; + } + + for (int i = startIndex; i < array.length; i++) { + if (booleanToFind == array[i]) { + return i; + } + } + + return -1; + } + + + /** + * 在数组中查找一个元素序列。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + *

+ * 起始索引小于0则看作0，超出数组长度的起始索引则返回-1。 + *

+ * + * @param array + * 要扫描的数组 + * @param arrayToFind + * 要查找的元素序列 + * @param startIndex + * 起始索引 + * + * @return 该元素序列在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int indexOf(boolean[] array, boolean[] arrayToFind, int startIndex) { + if ((array == null) || (arrayToFind == null)) { + return -1; + } + + int sourceLength = array.length; + int targetLength = arrayToFind.length; + + if (startIndex >= sourceLength) { + return (targetLength == 0) ? sourceLength : (-1); + } + + if (startIndex < 0) { + startIndex = 0; + } + + if (targetLength == 0) { + return startIndex; + } + + boolean first = arrayToFind[0]; + int i = startIndex; + int max = sourceLength - targetLength; + + startSearchForFirst: while (true) { + // 查找第一个元素 + while ((i <= max) && (array[i] != first)) { + i++; + } + + if (i > max) { + return -1; + } + + // 已经找到第一个元素，接着找 + int j = i + 1; + int end = (j + targetLength) - 1; + int k = 1; + + while (j < end) { + if (array[j++] != arrayToFind[k++]) { + i++; + + // 重新查找第一个元素 + continue startSearchForFirst; + } + } + + // 找到了 + return i; + } + } + + + /** + * 在数组中从末尾开始查找一个元素。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + * @param array + * 要扫描的数组 + * @param booleanToFind + * 要查找的元素 + * + * @return 该元素在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int lastIndexOf(boolean[] array, boolean booleanToFind) { + return lastIndexOf(array, booleanToFind, Integer.MAX_VALUE); + } + + + /** + * 在数组中从末尾开始查找一个元素序列。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + * @param array + * 要扫描的数组 + * @param arrayToFind + * 要查找的元素序列 + * + * @return 该元素序列在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int lastIndexOf(boolean[] array, boolean[] arrayToFind) { + return lastIndexOf(array, arrayToFind, Integer.MAX_VALUE); + } + + + /** + * 在数组中从末尾开始查找一个元素。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + *

+ * 起始索引小于0则返回-1，超出数组长度的起始索引则从数组末尾开始找。 + *

+ * + * @param array + * 要扫描的数组 + * @param booleanToFind + * 要查找的元素 + * @param startIndex + * 起始索引 + * + * @return 该元素在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int lastIndexOf(boolean[] array, boolean booleanToFind, int startIndex) { + if (array == null) { + return -1; + } + + if (startIndex < 0) { + return -1; + } + else if (startIndex >= array.length) { + startIndex = array.length - 1; + } + + for (int i = startIndex; i >= 0; i--) { + if (booleanToFind == array[i]) { + return i; + } + } + + return -1; + } + + + /** + * 在数组中从末尾开始查找一个元素序列。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + *

+ * 起始索引小于0则返回-1，超出数组长度的起始索引则从数组末尾开始找。 + *

+ * + * @param array + * 要扫描的数组 + * @param arrayToFind + * 要查找的元素序列 + * @param startIndex + * 起始索引 + * + * @return 该元素序列在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int lastIndexOf(boolean[] array, boolean[] arrayToFind, int startIndex) { + if ((array == null) || (arrayToFind == null)) { + return -1; + } + + int sourceLength = array.length; + int targetLength = arrayToFind.length; + + int rightIndex = sourceLength - targetLength; + + if (startIndex < 0) { + return -1; + } + + if (startIndex > rightIndex) { + startIndex = rightIndex; + } + + if (targetLength == 0) { + return startIndex; + } + + int lastIndex = targetLength - 1; + boolean last = arrayToFind[lastIndex]; + int min = targetLength - 1; + int i = min + startIndex; + + startSearchForLast: while (true) { + while ((i >= min) && (array[i] != last)) { + i--; + } + + if (i < min) { + return -1; + } + + int j = i - 1; + int start = j - (targetLength - 1); + int k = lastIndex - 1; + + while (j > start) { + if (array[j--] != arrayToFind[k--]) { + i--; + continue startSearchForLast; + } + } + + return start + 1; + } + } + + + /** + * 判断指定对象是否存在于指定数组中。 + * + *

+ * 如果数组为null则返回false。 + *

+ * + * @param array + * 要扫描的数组 + * @param booleanToFind + * 要查找的元素 + * + * @return 如果找到则返回true + */ + public static boolean contains(boolean[] array, boolean booleanToFind) { + return indexOf(array, booleanToFind) != -1; + } + + + /** + * 判断指定元素序列是否存在于指定数组中。 + * + *

+ * 如果数组为null则返回false。 + *

+ * + * @param array + * 要扫描的数组 + * @param arrayToFind + * 要查找的元素序列 + * + * @return 如果找到则返回true + */ + public static boolean contains(boolean[] array, boolean[] arrayToFind) { + return indexOf(array, arrayToFind) != -1; + } + + + /* + * ========================================================================== + * == + */ + /* 在数组中查找一个元素或一个元素序列。 */ + /* */ + /* 类型：char[] */ + /* + * ========================================================================== + * == + */ + + /** + * 在数组中查找一个元素。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + * @param array + * 要扫描的数组 + * @param charToFind + * 要查找的元素 + * + * @return 该元素在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int indexOf(char[] array, char charToFind) { + return indexOf(array, charToFind, 0); + } + + + /** + * 在数组中查找一个元素序列。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + * @param array + * 要扫描的数组 + * @param arrayToFind + * 要查找的元素序列 + * + * @return 该元素序列在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int indexOf(char[] array, char[] arrayToFind) { + return indexOf(array, arrayToFind, 0); + } + + + /** + * 在数组中查找一个元素。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + *

+ * 起始索引小于0则看作0，超出数组长度的起始索引则返回-1。 + *

+ * + * @param array + * 要扫描的数组 + * @param charToFind + * 要查找的元素 + * @param startIndex + * 起始索引 + * + * @return 该元素在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int indexOf(char[] array, char charToFind, int startIndex) { + if (array == null) { + return -1; + } + + if (startIndex < 0) { + startIndex = 0; + } + + for (int i = startIndex; i < array.length; i++) { + if (charToFind == array[i]) { + return i; + } + } + + return -1; + } + + + /** + * 在数组中查找一个元素序列。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + *

+ * 起始索引小于0则看作0，超出数组长度的起始索引则返回-1。 + *

+ * + * @param array + * 要扫描的数组 + * @param arrayToFind + * 要查找的元素序列 + * @param startIndex + * 起始索引 + * + * @return 该元素序列在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int indexOf(char[] array, char[] arrayToFind, int startIndex) { + if ((array == null) || (arrayToFind == null)) { + return -1; + } + + int sourceLength = array.length; + int targetLength = arrayToFind.length; + + if (startIndex >= sourceLength) { + return (targetLength == 0) ? sourceLength : (-1); + } + + if (startIndex < 0) { + startIndex = 0; + } + + if (targetLength == 0) { + return startIndex; + } + + char first = arrayToFind[0]; + int i = startIndex; + int max = sourceLength - targetLength; + + startSearchForFirst: while (true) { + // 查找第一个元素 + while ((i <= max) && (array[i] != first)) { + i++; + } + + if (i > max) { + return -1; + } + + // 已经找到第一个元素，接着找 + int j = i + 1; + int end = (j + targetLength) - 1; + int k = 1; + + while (j < end) { + if (array[j++] != arrayToFind[k++]) { + i++; + + // 重新查找第一个元素 + continue startSearchForFirst; + } + } + + // 找到了 + return i; + } + } + + + /** + * 在数组中从末尾开始查找一个元素。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + * @param array + * 要扫描的数组 + * @param charToFind + * 要查找的元素 + * + * @return 该元素在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int lastIndexOf(char[] array, char charToFind) { + return lastIndexOf(array, charToFind, Integer.MAX_VALUE); + } + + + /** + * 在数组中从末尾开始查找一个元素序列。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + * @param array + * 要扫描的数组 + * @param arrayToFind + * 要查找的元素序列 + * + * @return 该元素序列在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int lastIndexOf(char[] array, char[] arrayToFind) { + return lastIndexOf(array, arrayToFind, Integer.MAX_VALUE); + } + + + /** + * 在数组中从末尾开始查找一个元素。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + *

+ * 起始索引小于0则返回-1，超出数组长度的起始索引则从数组末尾开始找。 + *

+ * + * @param array + * 要扫描的数组 + * @param charToFind + * 要查找的元素 + * @param startIndex + * 起始索引 + * + * @return 该元素在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int lastIndexOf(char[] array, char charToFind, int startIndex) { + if (array == null) { + return -1; + } + + if (startIndex < 0) { + return -1; + } + else if (startIndex >= array.length) { + startIndex = array.length - 1; + } + + for (int i = startIndex; i >= 0; i--) { + if (charToFind == array[i]) { + return i; + } + } + + return -1; + } + + + /** + * 在数组中从末尾开始查找一个元素序列。 + * + *

+ * 如果未找到或数组为null则返回-1。 + *

+ * + *

+ * 起始索引小于0则返回-1，超出数组长度的起始索引则从数组末尾开始找。 + *

+ * + * @param array + * 要扫描的数组 + * @param arrayToFind + * 要查找的元素序列 + * @param startIndex + * 起始索引 + * + * @return 该元素序列在数组中的序号，如果数组为null或未找到，则返回-1。 + */ + public static int lastIndexOf(char[] array, char[] arrayToFind, int startIndex) { + if ((array == null) || (arrayToFind == null)) { + return -1; + } + + int sourceLength = array.length; + int targetLength = arrayToFind.length; + + int rightIndex = sourceLength - targetLength; + + if (startIndex < 0) { + return -1; + } + + if (startIndex > rightIndex) { + startIndex = rightIndex; + } + + if (targetLength == 0) { + return startIndex; + } + + int lastIndex = targetLength - 1; + char last = arrayToFind[lastIndex]; + int min = targetLength - 1; + int i = min + startIndex; + + startSearchForLast: while (true) { + while ((i >= min) && (array[i] != last)) { + i--; + } + + if (i < min) { + return -1; + } + + int j = i - 1; + int start = j - (targetLength - 1); + int k = lastIndex - 1; + + while (j > start) { + if (array[j--] != arrayToFind[k--]) { + i--; + continue startSearchForLast; + } + } + + return start + 1; + } + } + + + /** + * 判断指定对象是否存在于指定数组中。 + * + *

+ * 如果数组为null则返回false。 + *

+ * + * @param array + * 要扫描的数组 + * @param charToFind + * 要查找的元素 + * + * @return 如果找到则返回true + */ + public static boolean contains(char[] array, char charToFind) { + return indexOf(array, charToFind) != -1; + } + + + /** + * 判断指定元素序列是否存在于指定数组中。 + * + *

+ * 如果数组为null则返回false。 + *

+ * + * @param array + * 要扫描的数组 + * @param arrayToFind + * 要查找的元素序列 + * + * @return 如果找到则返回true + */ + public static boolean contains(char[] array, char[] arrayToFind) { + return indexOf(array, arrayToFind) != -1; + } + + + /* + * ========================================================================== + * == + */ + /* 将数组转换成易于阅读的字符串表示。 */ + /* */ + /* 支持多维数组。 */ + /* + * ========================================================================== + * == + */ + + /** + * 将数组转换成易于阅读的字符串表示。 + * + *

+ * 如果数组是null则返回[]，支持多维数组。 如果数组元素为null + * ，则显示<null>。 + * + * 

+     * ArrayUtil.toString(null)                              = "[]"
+     * ArrayUtil.toString(new int[] {1, 2, 3})               = "[1, 2, 3]"
+     * ArrayUtil.toString(new boolean[] {true, false, true}) = "[true, false, true]"
+     * ArrayUtil.toString(new Object[] {
+     *                       {1, 2, 3},  // 嵌套数组
+     *                       hello,      // 嵌套非数组
+     *                       null,       // 嵌套null
+     *                       {},         // 嵌套空数组
+     *                       {2, 3, 4}   // 嵌套数组
+     *                    })                                 = "[[1, 2, 3], hello, , [], [2, 3, 4]]"
+     *
+ * + *

+ * + * @param array + * 要转换的数组 + * + * @return 字符串表示，"[]"表示空数组或null + */ + public static String toString(Object array) { + return toString(array, "[]", ""); + } + + + /** + * 将数组转换成易于阅读的字符串表示。 + * + *

+ * 如果数组是null则返回指定字符串，支持多维数组。 如果数组元素为null，则显示 + * <null>。 + * + * 

+     * ArrayUtil.toString(null, "null")                              = "null"
+     * ArrayUtil.toString(new int[] {1, 2, 3}, "null")               = "[1, 2, 3]"
+     * ArrayUtil.toString(new boolean[] {true, false, true}, "null") = "[true, false, true]"
+     * ArrayUtil.toString(new Object[] {
+     *                       {1, 2, 3},  // 嵌套数组
+     *                       hello,      // 嵌套非数组
+     *                       null,       // 嵌套null
+     *                       {},         // 嵌套空数组
+     *                       {2, 3, 4}   // 嵌套数组
+     *                    }, "null")                                 = "[[1, 2, 3], hello, , [], [2, 3, 4]]"
+     *
+ * + *

+ * + * @param array + * 要转换的数组 + * @param nullArrayStr + * 如果数组是null，则返回此字符串 + * + * @return 字符串表示，或返回指定字符串表示null + */ + public static String toString(Object array, String nullArrayStr) { + return toString(array, nullArrayStr, ""); + } + + + /** + * 将数组转换成易于阅读的字符串表示。 + * + *

+ * 如果数组是null则返回指定字符串，支持多维数组。 如果数组元素为null，则显示指定字符串。 + * + * 

+     * ArrayUtil.toString(null, "null", "NULL")                              = "null"
+     * ArrayUtil.toString(new int[] {1, 2, 3}, "null", "NULL")               = "[1, 2, 3]"
+     * ArrayUtil.toString(new boolean[] {true, false, true}, "null", "NULL") = "[true, false, true]"
+     * ArrayUtil.toString(new Object[] {
+     *                       {1, 2, 3},  // 嵌套数组
+     *                       hello,      // 嵌套非数组
+     *                       null,       // 嵌套null
+     *                       {},         // 嵌套空数组
+     *                       {2, 3, 4}   // 嵌套数组
+     *                    }, "null", "NULL")                                 = "[[1, 2, 3], hello, NULL, [], [2, 3, 4]]"
+     *
+ * + *

+ * + * @param array + * 要转换的数组 + * @param nullArrayStr + * 如果数组是null，则返回此字符串 + * @param nullElementStr + * 如果数组中的元素为null，则返回此字符串 + * + * @return 字符串表示，或返回指定字符串表示null + */ + public static String toString(Object array, String nullArrayStr, String nullElementStr) { + if (array == null) { + return nullArrayStr; + } + + StringBuffer buffer = new StringBuffer(); + + toString(buffer, array, nullArrayStr, nullElementStr); + + return buffer.toString(); + } + + + /** + * 将数组转换成易于阅读的字符串表示。null将被看作空数组。 支持多维数组。 + * + * @param buffer + * 将转换后的字符串加入到这个StringBuffer中 + * @param array + * 要转换的数组 + * @param nullArrayStr + * 如果数组是null，则返回此字符串 + * @param nullElementStr + * 如果数组中的元素为null，则返回此字符串 + */ + private static void toString(StringBuffer buffer, Object array, String nullArrayStr, String nullElementStr) { + if (array == null) { + buffer.append(nullElementStr); + return; + } + + if (!array.getClass().isArray()) { + buffer.append(ObjectUtil.toString(array, nullElementStr)); + return; + } + + buffer.append('['); + + // array为数组 + if (array instanceof long[]) { + long[] longArray = (long[]) array; + int length = longArray.length; + + for (int i = 0; i < length; i++) { + if (i > 0) { + buffer.append(", "); + } + + buffer.append(longArray[i]); + } + } + else if (array instanceof int[]) { + int[] intArray = (int[]) array; + int length = intArray.length; + + for (int i = 0; i < length; i++) { + if (i > 0) { + buffer.append(", "); + } + + buffer.append(intArray[i]); + } + } + else if (array instanceof short[]) { + short[] shortArray = (short[]) array; + int length = shortArray.length; + + for (int i = 0; i < length; i++) { + if (i > 0) { + buffer.append(", "); + } + + buffer.append(shortArray[i]); + } + } + else if (array instanceof byte[]) { + byte[] byteArray = (byte[]) array; + int length = byteArray.length; + + for (int i = 0; i < length; i++) { + if (i > 0) { + buffer.append(", "); + } + else { + buffer.append("0x"); + } + + String hexStr = Integer.toHexString(0xFF & byteArray[i]).toUpperCase(); + + if (hexStr.length() == 0) { + buffer.append("00"); + } + else if (hexStr.length() == 1) { + buffer.append("0"); + } + + buffer.append(hexStr); + } + } + else if (array instanceof double[]) { + double[] doubleArray = (double[]) array; + int length = doubleArray.length; + + for (int i = 0; i < length; i++) { + if (i > 0) { + buffer.append(", "); + } + + buffer.append(doubleArray[i]); + } + } + else if (array instanceof float[]) { + float[] floatArray = (float[]) array; + int length = floatArray.length; + + for (int i = 0; i < length; i++) { + if (i > 0) { + buffer.append(", "); + } + + buffer.append(floatArray[i]); + } + } + else if (array instanceof boolean[]) { + boolean[] booleanArray = (boolean[]) array; + int length = booleanArray.length; + + for (int i = 0; i < length; i++) { + if (i > 0) { + buffer.append(", "); + } + + buffer.append(booleanArray[i]); + } + } + else if (array instanceof char[]) { + char[] charArray = (char[]) array; + int length = charArray.length; + + for (int i = 0; i < length; i++) { + if (i > 0) { + buffer.append(", "); + } + + buffer.append(charArray[i]); + } + } + else { + Object[] objectArray = (Object[]) array; + int length = objectArray.length; + + for (int i = 0; i < length; i++) { + if (i > 0) { + buffer.append(", "); + } + + toString(buffer, objectArray[i], nullArrayStr, nullElementStr); + } + } + + buffer.append(']'); + } +} diff --git a/rocketmq-filtersrv/src/main/java/com/alibaba/common/lang/ClassInstantiationException.java b/rocketmq-filtersrv/src/main/java/com/alibaba/common/lang/ClassInstantiationException.java new file mode 100644 index 000000000..39e195c95 --- /dev/null +++ b/rocketmq-filtersrv/src/main/java/com/alibaba/common/lang/ClassInstantiationException.java @@ -0,0 +1,58 @@ +package com.alibaba.common.lang; + +import com.alibaba.common.lang.exception.ChainedException; + + +/** + * 代表实例化类时失败的异常。 + * + * @author Michael Zhou + * @version $Id: ClassInstantiationException.java 1291 2005-03-04 03:23:30Z + * baobao $ + */ +public class ClassInstantiationException extends ChainedException { + private static final long serialVersionUID = 3258408422113555761L; + + + /** + * 构造一个空的异常. + */ + public ClassInstantiationException() { + super(); + } + + + /** + * 构造一个异常, 指明异常的详细信息. + * + * @param message + * 详细信息 + */ + public ClassInstantiationException(String message) { + super(message); + } + + + /** + * 构造一个异常, 指明引起这个异常的起因. + * + * @param cause + * 异常的起因 + */ + public ClassInstantiationException(Throwable cause) { + super(cause); + } + + + /** + * 构造一个异常, 指明引起这个异常的起因. + * + * @param message + * 详细信息 + * @param cause + * 异常的起因 + */ + public ClassInstantiationException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/rocketmq-filtersrv/src/main/java/com/alibaba/common/lang/ClassUtil.java b/rocketmq-filtersrv/src/main/java/com/alibaba/common/lang/ClassUtil.java new file mode 100644 index 000000000..660e731b6 --- /dev/null +++ b/rocketmq-filtersrv/src/main/java/com/alibaba/common/lang/ClassUtil.java @@ -0,0 +1,1367 @@ +package com.alibaba.common.lang; + +import java.lang.reflect.Array; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.WeakHashMap; + + +/** + * 有关 Class 处理的工具类。 + * + *

+ * 这个类中的每个方法都可以“安全”地处理 null ，而不会抛出 + * NullPointerException。 + *

+ * + * @author Michael Zhou + * @version $Id: ClassUtil.java 509 2004-02-16 05:42:07Z baobao $ + */ +public class ClassUtil { + /* + * ========================================================================== + * == + */ + /* 常量和singleton。 */ + /* + * ========================================================================== + * == + */ + + /** 资源文件的分隔符： '/'。 */ + public static final char RESOURCE_SEPARATOR_CHAR = '/'; + + /** Java类名的分隔符： '.'。 */ + public static final char PACKAGE_SEPARATOR_CHAR = '.'; + + /** Java类名的分隔符： "."。 */ + public static final String PACKAGE_SEPARATOR = String.valueOf(PACKAGE_SEPARATOR_CHAR); + + /** 内联类的分隔符： '$'。 */ + public static final char INNER_CLASS_SEPARATOR_CHAR = '$'; + + /** 内联类的分隔符： "$"。 */ + public static final String INNER_CLASS_SEPARATOR = String.valueOf(INNER_CLASS_SEPARATOR_CHAR); + + /** 所有类的信息表，包括父类, 接口, 数组的维数等信息。 */ + private static Map TYPE_MAP = Collections.synchronizedMap(new WeakHashMap()); + + + /* + * ========================================================================== + * == + */ + /* 取得类名和package名的方法。 */ + /* + * ========================================================================== + * == + */ + + /** + * 取得对象所属的类的直观类名。 + * + *

+ * 相当于 object.getClass().getName() ，但不同的是，该方法用更直观的方式显示数组类型。 例如： + * + * 

+     *  int[].class.getName() = "[I" ClassUtil.getClassName(int[].class) = "int[]"
+     * 
+     *  Integer[][].class.getName() = "[[Ljava.lang.Integer;" ClassUtil.getClassName(Integer[][].class) = "java.lang.Integer[][]"
+     *
+ * + *

+ * + *

+ * 对于非数组的类型，该方法等效于 Class.getName() 方法。 + *

+ * + *

+ * 注意，该方法所返回的数组类名只能用于显示给人看，不能用于 Class.forName 操作。 + *

+ * + * @param object + * 要显示类名的对象 + * + * @return 用于显示的直观类名，如果原类名为空或非法，则返回 null + */ + public static String getClassNameForObject(Object object) { + if (object == null) { + return null; + } + + return getClassName(object.getClass().getName(), true); + } + + + /** + * 取得直观的类名。 + * + *

+ * 相当于 clazz.getName() ，但不同的是，该方法用更直观的方式显示数组类型。 例如： + * + * 

+     *  int[].class.getName() = "[I" ClassUtil.getClassName(int[].class) = "int[]"
+     * 
+     *  Integer[][].class.getName() = "[[Ljava.lang.Integer;" ClassUtil.getClassName(Integer[][].class) = "java.lang.Integer[][]"
+     *
+ * + *

+ * + *

+ * 对于非数组的类型，该方法等效于 Class.getName() 方法。 + *

+ * + *

+ * 注意，该方法所返回的数组类名只能用于显示给人看，不能用于 Class.forName 操作。 + *

+ * + * @param clazz + * 要显示类名的类 + * + * @return 用于显示的直观类名，如果原始类为 null ，则返回 null + */ + public static String getClassName(Class clazz) { + if (clazz == null) { + return null; + } + + return getClassName(clazz.getName(), true); + } + + + /** + * 取得直观的类名。 + * + *

+ * className 必须是从 clazz.getName() + * 所返回的合法类名。该方法用更直观的方式显示数组类型。 例如： + * + * 

+     *  int[].class.getName() = "[I" ClassUtil.getClassName(int[].class) = "int[]"
+     * 
+     *  Integer[][].class.getName() = "[[Ljava.lang.Integer;" ClassUtil.getClassName(Integer[][].class) = "java.lang.Integer[][]"
+     *
+ * + *

+ * + *

+ * 对于非数组的类型，该方法等效于 Class.getName() 方法。 + *

+ * + *

+ * 注意，该方法所返回的数组类名只能用于显示给人看，不能用于 Class.forName 操作。 + *

+ * + * @param className + * 要显示的类名 + * + * @return 用于显示的直观类名，如果原类名为 null ，则返回 null + * ，如果原类名是非法的，则返回原类名 + */ + public static String getClassName(String className) { + return getClassName(className, true); + } + + + /** + * 取得直观的类名。 + * + * @param className + * 类名 + * @param processInnerClass + * 是否将内联类分隔符 '$' 转换成 '.' + * + * @return 直观的类名，或 null + */ + private static String getClassName(String className, boolean processInnerClass) { + if (StringUtil.isEmpty(className)) { + return className; + } + + if (processInnerClass) { + className = className.replace(INNER_CLASS_SEPARATOR_CHAR, PACKAGE_SEPARATOR_CHAR); + } + + int length = className.length(); + int dimension = 0; + + // 取得数组的维数，如果不是数组，维数为0 + for (int i = 0; i < length; i++, dimension++) { + if (className.charAt(i) != '[') { + break; + } + } + + // 如果不是数组，则直接返回 + if (dimension == 0) { + return className; + } + + // 确保类名合法 + if (length <= dimension) { + return className; // 非法类名 + } + + // 处理数组 + StringBuffer componentTypeName = new StringBuffer(); + + switch (className.charAt(dimension)) { + case 'Z': + componentTypeName.append("boolean"); + break; + + case 'B': + componentTypeName.append("byte"); + break; + + case 'C': + componentTypeName.append("char"); + break; + + case 'D': + componentTypeName.append("double"); + break; + + case 'F': + componentTypeName.append("float"); + break; + + case 'I': + componentTypeName.append("int"); + break; + + case 'J': + componentTypeName.append("long"); + break; + + case 'S': + componentTypeName.append("short"); + break; + + case 'L': + + if ((className.charAt(length - 1) != ';') || (length <= (dimension + 2))) { + return className; // 非法类名 + } + + componentTypeName.append(className.substring(dimension + 1, length - 1)); + break; + + default: + return className; // 非法类名 + } + + for (int i = 0; i < dimension; i++) { + componentTypeName.append("[]"); + } + + return componentTypeName.toString(); + } + + + /** + * 取得指定对象所属的类的短类名，不包括package名。 + * + *

+ * 此方法可以正确显示数组和内联类的名称。 + *

+ * + *

+ * 例如： + * + * 

+     *  ClassUtil.getShortClassNameForObject(Boolean.TRUE) = "Boolean" ClassUtil.getShortClassNameForObject(new Boolean[10]) = "Boolean[]" ClassUtil.getShortClassNameForObject(new int[1][2]) = "int[][]"
+     *
+ * + *

+ * + * @param object + * 要查看的对象 + * + * @return 短类名，如果对象为 null ，则返回 null + */ + public static String getShortClassNameForObject(Object object) { + if (object == null) { + return null; + } + + return getShortClassName(object.getClass().getName()); + } + + + /** + * 取得短类名，不包括package名。 + * + *

+ * 此方法可以正确显示数组和内联类的名称。 + *

+ * + *

+ * 例如： + * + * 

+     *  ClassUtil.getShortClassName(Boolean.class) = "Boolean" ClassUtil.getShortClassName(Boolean[].class) = "Boolean[]" ClassUtil.getShortClassName(int[][].class) = "int[][]" ClassUtil.getShortClassName(Map.Entry.class) = "Map.Entry"
+     *
+ * + *

+ * + * @param clazz + * 要查看的类 + * + * @return 短类名，如果类为 null ，则返回 null + */ + public static String getShortClassName(Class clazz) { + if (clazz == null) { + return null; + } + + return getShortClassName(clazz.getName()); + } + + + /** + * 取得类名，不包括package名。 + * + *

+ * 此方法可以正确显示数组和内联类的名称。 + *

+ * + *

+ * 例如： + * + * 

+     *  ClassUtil.getShortClassName(Boolean.class.getName()) = "Boolean" ClassUtil.getShortClassName(Boolean[].class.getName()) = "Boolean[]" ClassUtil.getShortClassName(int[][].class.getName()) = "int[][]" ClassUtil.getShortClassName(Map.Entry.class.getName()) = "Map.Entry"
+     *
+ * + *

+ * + * @param className + * 要查看的类名 + * + * @return 短类名，如果类名为空，则返回 null + */ + public static String getShortClassName(String className) { + if (StringUtil.isEmpty(className)) { + return className; + } + + // 转换成直观的类名 + className = getClassName(className, false); + + char[] chars = className.toCharArray(); + int lastDot = 0; + + for (int i = 0; i < chars.length; i++) { + if (chars[i] == PACKAGE_SEPARATOR_CHAR) { + lastDot = i + 1; + } + else if (chars[i] == INNER_CLASS_SEPARATOR_CHAR) { + chars[i] = PACKAGE_SEPARATOR_CHAR; + } + } + + return new String(chars, lastDot, chars.length - lastDot); + } + + + /** + * 取得指定对象所属的类的package名。 + * + *

+ * 对于数组，此方法返回的是数组元素类型的package名。 + *

+ * + * @param object + * 要查看的对象 + * + * @return package名，如果对象为 null ，则返回 null + */ + public static String getPackageNameForObject(Object object) { + if (object == null) { + return null; + } + + return getPackageName(object.getClass().getName()); + } + + + /** + * 取得指定类的package名。 + * + *

+ * 对于数组，此方法返回的是数组元素类型的package名。 + *

+ * + * @param clazz + * 要查看的类 + * + * @return package名，如果类为 null ，则返回 null + */ + public static String getPackageName(Class clazz) { + if (clazz == null) { + return null; + } + + return getPackageName(clazz.getName()); + } + + + /** + * 取得指定类名的package名。 + * + *

+ * 对于数组，此方法返回的是数组元素类型的package名。 + *

+ * + * @param className + * 要查看的类名 + * + * @return package名，如果类名为空，则返回 null + */ + public static String getPackageName(String className) { + if (StringUtil.isEmpty(className)) { + return null; + } + + // 转换成直观的类名 + className = getClassName(className, false); + + int i = className.lastIndexOf(PACKAGE_SEPARATOR_CHAR); + + if (i == -1) { + return ""; + } + + return className.substring(0, i); + } + + + /* + * ========================================================================== + * == + */ + /* 取得类名和package名的resource名的方法。 */ + /* */ + /* 和类名、package名不同的是，resource名符合文件名命名规范，例如： */ + /* java/lang/String.class */ + /* com/alibaba/commons/lang */ + /* etc. */ + /* + * ========================================================================== + * == + */ + + /** + * 取得对象所属的类的资源名。 + * + *

+ * 例如： + * + * 

+     * ClassUtil.getClassNameForObjectAsResource("This is a string") = "java/lang/String.class"
+     *
+ * + *

+ * + * @param object + * 要显示类名的对象 + * + * @return 指定对象所属类的资源名，如果对象为空，则返回null + */ + public static String getClassNameForObjectAsResource(Object object) { + if (object == null) { + return null; + } + + return object.getClass().getName().replace(PACKAGE_SEPARATOR_CHAR, RESOURCE_SEPARATOR_CHAR) + + ".class"; + } + + + /** + * 取得指定类的资源名。 + * + *

+ * 例如： + * + * 

+     * ClassUtil.getClassNameAsResource(String.class) = "java/lang/String.class"
+     *
+ * + *

+ * + * @param clazz + * 要显示类名的类 + * + * @return 指定类的资源名，如果指定类为空，则返回null + */ + public static String getClassNameAsResource(Class clazz) { + if (clazz == null) { + return null; + } + + return clazz.getName().replace(PACKAGE_SEPARATOR_CHAR, RESOURCE_SEPARATOR_CHAR) + ".class"; + } + + + /** + * 取得指定类的资源名。 + * + *

+ * 例如： + * + * 

+     * ClassUtil.getClassNameAsResource("java.lang.String") = "java/lang/String.class"
+     *
+ * + *

+ * + * @param className + * 要显示的类名 + * + * @return 指定类名对应的资源名，如果指定类名为空，则返回null + */ + public static String getClassNameAsResource(String className) { + if (className == null) { + return null; + } + + return className.replace(PACKAGE_SEPARATOR_CHAR, RESOURCE_SEPARATOR_CHAR) + ".class"; + } + + + /** + * 取得指定对象所属的类的package名的资源名。 + * + *

+ * 对于数组，此方法返回的是数组元素类型的package名。 + *

+ * + * @param object + * 要查看的对象 + * + * @return package名，如果对象为 null ，则返回 null + */ + public static String getPackageNameForObjectAsResource(Object object) { + if (object == null) { + return null; + } + + return getPackageNameForObject(object).replace(PACKAGE_SEPARATOR_CHAR, RESOURCE_SEPARATOR_CHAR); + } + + + /** + * 取得指定类的package名的资源名。 + * + *

+ * 对于数组，此方法返回的是数组元素类型的package名。 + *

+ * + * @param clazz + * 要查看的类 + * + * @return package名，如果类为 null ，则返回 null + */ + public static String getPackageNameAsResource(Class clazz) { + if (clazz == null) { + return null; + } + + return getPackageName(clazz).replace(PACKAGE_SEPARATOR_CHAR, RESOURCE_SEPARATOR_CHAR); + } + + + /** + * 取得指定类名的package名的资源名。 + * + *

+ * 对于数组，此方法返回的是数组元素类型的package名。 + *

+ * + * @param className + * 要查看的类名 + * + * @return package名，如果类名为空，则返回 null + */ + public static String getPackageNameAsResource(String className) { + if (className == null) { + return null; + } + + return getPackageName(className).replace(PACKAGE_SEPARATOR_CHAR, RESOURCE_SEPARATOR_CHAR); + } + + + /* + * ========================================================================== + * == + */ + /* 取得类的信息，如父类, 接口, 数组的维数等。 */ + /* + * ========================================================================== + * == + */ + + /** + * 取得指定维数的 Array类. + * + * @param componentType + * 数组的基类 + * @param dimension + * 维数，如果小于 0 则看作 0 + * + * @return 如果维数为0, 则返回基类本身, 否则返回数组类，如果数组的基类为 null ，则返回 + * null + */ + public static Class getArrayClass(Class componentType, int dimension) { + if (dimension <= 0) { + return componentType; + } + + if (componentType == null) { + return null; + } + + return Array.newInstance(componentType, new int[dimension]).getClass(); + } + + + /** + * 取得数组元素的类型。 + * + * @param type + * 要查找的类 + * + * @return 如果是数组, 则返回数组元素的类型, 否则返回 null + */ + public static Class getArrayComponentType(Class type) { + if (type == null) { + return null; + } + + return getTypeInfo(type).getArrayComponentType(); + } + + + /** + * 取得数组的维数。 + * + * @param clazz + * 要查找的类 + * + * @return 数组的维数. 如果不是数组, 则返回 0 ，如果数组为 null ，是返回 + * -1 + */ + public static int getArrayDimension(Class clazz) { + if (clazz == null) { + return -1; + } + + return getTypeInfo(clazz).getArrayDimension(); + } + + + /** + * 取得指定类的所有父类。 + * + *

+ * 对于一个 Class 实例，如果它不是接口，也不是数组，此方法依次列出从该类的父类开始直到 + * Object 的所有类。 + *

+ * + *

+ * 例如 ClassUtil.getSuperclasses(java.util.ArrayList.class) + * 返回以下列表： + * + *

    + *
  1. + * java.util.AbstractList
    2. + *
  2. + * java.util.AbstractCollection
    3. + *
  3. + * java.lang.Object
    4. + *
+ *

+ * + *

+ * 对于一个接口，此方法返回一个空列表。 + *

+ * + *

+ * 例如ClassUtil.getSuperclasses(java.util.List.class)将返回一个空列表。 + *

+ * + *

+ * 对于一个数组，此方法返回一个列表，列出所有component类型的父类的相同维数的数组类型。 例如： + * ClassUtil.getSuperclasses(java.util.ArrayList[][].class) + * 返回以下列表： + * + *

    + *
  1. + * java.util.AbstractList[][]
    2. + *
  2. + * java.util.AbstractCollection[][]
    3. + *
  3. + * java.lang.Object[][]
    4. + *
  4. + * java.lang.Object[]
    5. + *
  5. + * java.lang.Object
    6. + *
+ * + * 注意，原子类型及其数组，将被转换成相应的包装类来处理。 例如： + * ClassUtil.getSuperclasses(int[][].class) 返回以下列表： + * + *
    + *
  1. + * java.lang.Number[][]
    2. + *
  2. + * java.lang.Object[][]
    3. + *
  3. + * java.lang.Object[]
    4. + *
  4. + * java.lang.Object
    5. + *
+ *

+ * + * @param clazz + * 要查找的类 + * + * @return 所有父类的列表，如果指定类为 null ，则返回 null + */ + public static List getSuperclasses(Class clazz) { + if (clazz == null) { + return null; + } + + return getTypeInfo(clazz).getSuperclasses(); + } + + + /** + * 取得指定类的所有接口。 + * + *

+ * 对于一个 Class 实例，如果它不是接口，也不是数组，此方法依次列出从该类的父类开始直到 + * Object 的所有类。 + *

+ * + *

+ * 例如 ClassUtil.getInterfaces(java.util.ArrayList.class) + * 返回以下列表： + * + *

    + *
  1. + * java.util.List
    2. + *
  2. + * java.util.Collection
    3. + *
  3. + * java.util.RandomAccess
    4. + *
  4. + * java.lang.Cloneable
    5. + *
  5. + * java.io.Serializable
    6. + *
+ *

+ * + *

+ * 对于一个数组，此方法返回一个列表，列出所有component类型的接口的相同维数的数组类型。 例如： + * ClassUtil.getInterfaces(java.util.ArrayList[][].class) + * 返回以下列表： + * + *

    + *
  1. + * java.util.List[][]
    2. + *
  2. + * java.util.Collection[][]
    3. + *
  3. + * java.util.RandomAccess[][]
    4. + *
  4. + * java.lang.Cloneable[][]
    5. + *
  5. + * java.io.Serializable[][]
    6. + *
+ *

+ * + *

+ * 注意，原子类型及其数组，将被转换成相应的包装类来处理。 例如： + * ClassUtil.getInterfaces(int[][].class) 返回以下列表： + * + *

    + *
  1. + * java.lang.Comparable[][]
    2. + *
  2. + * java.io.Serializable[][]
    3. + *
+ *

+ * + * @param clazz + * 要查找的类 + * + * @return 所有接口的列表，如果指定类为 null ，则返回 null + */ + public static List getInterfaces(Class clazz) { + if (clazz == null) { + return null; + } + + return getTypeInfo(clazz).getInterfaces(); + } + + + /** + * 判断指定类是否为内联类。 + * + * @param clazz + * 要查找的类 + * + * @return 如果是，则返回 true + */ + public static boolean isInnerClass(Class clazz) { + if (clazz == null) { + return false; + } + + return StringUtil.contains(clazz.getName(), INNER_CLASS_SEPARATOR_CHAR); + } + + + /** + * 检查一组指定类型 fromClasses 的对象是否可以赋值给另一组类型 classes。 + * + *

+ * 此方法可以用来确定指定类型的参数 object1, object2, ... 是否可以用来调用确定参数类型为 + * class1, class2, + * ... 的方法。 + *

+ * + *

+ * 对于 fromClasses 的每个元素 fromClass 和 + * classes 的每个元素 clazz， 按照如下规则： + * + *

    + *
  1. + * 如果目标类 clazz 为 null ，总是返回 false。
    2. + *
  2. + * 如果参数类型 fromClass 为 null ，并且目标类型 + * clazz 为非原子类型，则返回 true。 因为 null + * 可以被赋给任何引用类型。
    3. + *
  3. + * 调用 Class.isAssignableFrom 方法来确定目标类 clazz 是否和参数类 + * fromClass 相同或是其父类、接口，如果是，则返回 true。
    4. + *
  4. + * 如果目标类型 clazz 为原子类型，那么根据 The Java Language + * Specification ，sections 5.1.1, 5.1.2, 5.1.4定义的Widening Primitive + * Conversion规则，参数类型 fromClass 可以是任何能扩展成该目标类型的原子类型及其包装类。 例如， + * clazz 为 long ，那么参数类型可以是 byte、 + * short、int、long、char + * 及其包装类 java.lang.Byte、java.lang.Short、 + * java.lang.Integer、 java.lang.Long 和 + * java.lang.Character 。如果满足这个条件，则返回 true。
    5. + *
  5. + * 不满足上述所有条件，则返回 false。
    6. + *
+ *

+ * + * @param classes + * 目标类型列表，如果是 null 总是返回 false + * @param fromClasses + * 参数类型列表， null 表示可赋值给任意非原子类型 + * + * @return 如果可以被赋值，则返回 true + */ + public static boolean isAssignable(Class[] classes, Class[] fromClasses) { + if (!ArrayUtil.isSameLength(fromClasses, classes)) { + return false; + } + + if (fromClasses == null) { + fromClasses = ArrayUtil.EMPTY_CLASS_ARRAY; + } + + if (classes == null) { + classes = ArrayUtil.EMPTY_CLASS_ARRAY; + } + + for (int i = 0; i < fromClasses.length; i++) { + if (isAssignable(classes[i], fromClasses[i]) == false) { + return false; + } + } + + return true; + } + + + /** + * 检查指定类型 fromClass 的对象是否可以赋值给另一种类型 clazz。 + * + *

+ * 此方法可以用来确定指定类型的参数 object1, object2, ... 是否可以用来调用确定参数类型 + * class1, class2, + * ... 的方法。 + *

+ * + *

+ * 按照如下规则： + * + *

    + *
  1. + * 如果目标类 clazz 为 null ，总是返回 false。
    2. + *
  2. + * 如果参数类型 fromClass 为 null ，并且目标类型 + * clazz 为非原子类型，则返回 true。 因为 null + * 可以被赋给任何引用类型。
    3. + *
  3. + * 调用 Class.isAssignableFrom 方法来确定目标类 clazz 是否和参数类 + * fromClass 相同或是其父类、接口，如果是，则返回 true。
    4. + *
  4. + * 如果目标类型 clazz 为原子类型，那么根据 The Java Language + * Specification ，sections 5.1.1, 5.1.2, 5.1.4定义的Widening Primitive + * Conversion规则，参数类型 fromClass 可以是任何能扩展成该目标类型的原子类型及其包装类。 例如， + * clazz 为 long ，那么参数类型可以是 byte、 + * short、int、long、char + * 及其包装类 java.lang.Byte、java.lang.Short、 + * java.lang.Integer、 java.lang.Long 和 + * java.lang.Character 。如果满足这个条件，则返回 true。
    5. + *
  5. + * 不满足上述所有条件，则返回 false。
    6. + *
+ *

+ * + * @param clazz + * 目标类型，如果是 null 总是返回 false + * @param fromClass + * 参数类型， null 表示可赋值给任意非原子类型 + * + * @return 如果可以被赋值，则返回 null + */ + public static boolean isAssignable(Class clazz, Class fromClass) { + if (clazz == null) { + return false; + } + + // 如果fromClass是null，只要clazz不是原子类型如int，就一定可以赋值 + if (fromClass == null) { + return !clazz.isPrimitive(); + } + + // 如果类相同或有父子关系，当然可以赋值 + if (clazz.isAssignableFrom(fromClass)) { + return true; + } + + // 对于原子类型，根据JLS的规则进行扩展 + // 目标class为原子类型时，fromClass可以为原子类型和原子类型的包装类型。 + if (clazz.isPrimitive()) { + // boolean可以接受：boolean + if (Boolean.TYPE.equals(clazz)) { + return Boolean.class.equals(fromClass); + } + + // byte可以接受：byte + if (Byte.TYPE.equals(clazz)) { + return Byte.class.equals(fromClass); + } + + // char可以接受：char + if (Character.TYPE.equals(clazz)) { + return Character.class.equals(fromClass); + } + + // short可以接受：short, byte + if (Short.TYPE.equals(clazz)) { + return Short.class.equals(fromClass) || Byte.TYPE.equals(fromClass) + || Byte.class.equals(fromClass); + } + + // int可以接受：int、byte、short、char + if (Integer.TYPE.equals(clazz)) { + return Integer.class.equals(fromClass) || Byte.TYPE.equals(fromClass) + || Byte.class.equals(fromClass) || Short.TYPE.equals(fromClass) + || Short.class.equals(fromClass) || Character.TYPE.equals(fromClass) + || Character.class.equals((fromClass)); + } + + // long可以接受：long、int、byte、short、char + if (Long.TYPE.equals(clazz)) { + return Long.class.equals(fromClass) || Integer.TYPE.equals(fromClass) + || Integer.class.equals(fromClass) || Byte.TYPE.equals(fromClass) + || Byte.class.equals(fromClass) || Short.TYPE.equals(fromClass) + || Short.class.equals(fromClass) || Character.TYPE.equals(fromClass) + || Character.class.equals((fromClass)); + } + + // float可以接受：float, long, int, byte, short, char + if (Float.TYPE.equals(clazz)) { + return Float.class.equals(fromClass) || Long.TYPE.equals(fromClass) + || Long.class.equals(fromClass) || Integer.TYPE.equals(fromClass) + || Integer.class.equals(fromClass) || Byte.TYPE.equals(fromClass) + || Byte.class.equals(fromClass) || Short.TYPE.equals(fromClass) + || Short.class.equals(fromClass) || Character.TYPE.equals(fromClass) + || Character.class.equals((fromClass)); + } + + // double可以接受：double, float, long, int, byte, short, char + if (Double.TYPE.equals(clazz)) { + return Double.class.equals(fromClass) || Float.TYPE.equals(fromClass) + || Float.class.equals(fromClass) || Long.TYPE.equals(fromClass) + || Long.class.equals(fromClass) || Integer.TYPE.equals(fromClass) + || Integer.class.equals(fromClass) || Byte.TYPE.equals(fromClass) + || Byte.class.equals(fromClass) || Short.TYPE.equals(fromClass) + || Short.class.equals(fromClass) || Character.TYPE.equals(fromClass) + || Character.class.equals((fromClass)); + } + } + + return false; + } + + + /** + * 取得指定类的 TypeInfo。 + * + * @param type + * 指定类或接口 + * + * @return TypeInfo 对象. + */ + protected static TypeInfo getTypeInfo(Class type) { + if (type == null) { + throw new IllegalArgumentException("Parameter clazz should not be null"); + } + + TypeInfo classInfo; + + synchronized (TYPE_MAP) { + classInfo = (TypeInfo) TYPE_MAP.get(type); + + if (classInfo == null) { + classInfo = new TypeInfo(type); + TYPE_MAP.put(type, classInfo); + } + } + + return classInfo; + } + + /** + * 代表一个类的信息, 包括父类, 接口, 数组的维数等. + */ + protected static class TypeInfo { + private Class type; + private Class componentType; + private int dimension; + private List superclasses = new ArrayList(2); + private List interfaces = new ArrayList(2); + + + /** + * 创建 TypeInfo。 + * + * @param type + * 创建指定类的 TypeInfo + */ + private TypeInfo(Class type) { + this.type = type; + + // 如果是array, 设置componentType和dimension + Class componentType = null; + + if (type.isArray()) { + componentType = type; + + do { + componentType = componentType.getComponentType(); + dimension++; + } while (componentType.isArray()); + } + + this.componentType = componentType; + + // 取得所有superclass + if (dimension > 0) { + // 将primitive类型转换成对应的包装类 + componentType = getNonPrimitiveType(componentType); + + Class superComponentType = componentType.getSuperclass(); + + // 如果是primitive, interface, 则设置其基类为Object. + if ((superComponentType == null) && !Object.class.equals(componentType)) { + superComponentType = Object.class; + } + + if (superComponentType != null) { + Class superclass = getArrayClass(superComponentType, dimension); + + superclasses.add(superclass); + superclasses.addAll(getTypeInfo(superclass).superclasses); + } + else { + for (int i = dimension - 1; i >= 0; i--) { + superclasses.add(getArrayClass(Object.class, i)); + } + } + } + else { + // 将primitive类型转换成对应的包装类 + type = getNonPrimitiveType(type); + + Class superclass = type.getSuperclass(); + + if (superclass != null) { + superclasses.add(superclass); + superclasses.addAll(getTypeInfo(superclass).superclasses); + } + } + + // 取得所有interface + if (dimension == 0) { + Class[] typeInterfaces = type.getInterfaces(); + List set = new ArrayList(); + + for (int i = 0; i < typeInterfaces.length; i++) { + Class typeInterface = typeInterfaces[i]; + + set.add(typeInterface); + set.addAll(getTypeInfo(typeInterface).interfaces); + } + + for (Iterator i = superclasses.iterator(); i.hasNext();) { + Class typeInterface = (Class) i.next(); + + set.addAll(getTypeInfo(typeInterface).interfaces); + } + + for (Iterator i = set.iterator(); i.hasNext();) { + Class interfaceClass = (Class) i.next(); + + if (!interfaces.contains(interfaceClass)) { + interfaces.add(interfaceClass); + } + } + } + else { + for (Iterator i = getTypeInfo(componentType).interfaces.iterator(); i.hasNext();) { + Class componentInterface = (Class) i.next(); + + interfaces.add(getArrayClass(componentInterface, dimension)); + } + } + } + + + /** + * 将所有的原子类型转换成对应的包装类，其它类型不变。 + * + * @param type + * 要转换的类型 + * + * @return 非原子类型 + */ + private Class getNonPrimitiveType(Class type) { + if (type.isPrimitive()) { + if (Integer.TYPE.equals(type)) { + type = Integer.class; + } + else if (Long.TYPE.equals(type)) { + type = Long.class; + } + else if (Short.TYPE.equals(type)) { + type = Short.class; + } + else if (Byte.TYPE.equals(type)) { + type = Byte.class; + } + else if (Float.TYPE.equals(type)) { + type = Float.class; + } + else if (Double.TYPE.equals(type)) { + type = Double.class; + } + else if (Boolean.TYPE.equals(type)) { + type = Boolean.class; + } + else if (Character.TYPE.equals(type)) { + type = Character.class; + } + } + + return type; + } + + + /** + * 取得 TypeInfo 所代表的java类。 + * + * @return TypeInfo 所代表的java类 + */ + public Class getType() { + return type; + } + + + /** + * 取得数组元素的类型。 + * + * @return 如果是数组, 则返回数组元素的类型, 否则返回 null + */ + public Class getArrayComponentType() { + return componentType; + } + + + /** + * 取得数组的维数。 + * + * @return 数组的维数. 如果不是数组, 则返回 0 + */ + public int getArrayDimension() { + return dimension; + } + + + /** + * 取得所有的父类。 + * + * @return 所有的父类 + */ + public List getSuperclasses() { + return Collections.unmodifiableList(superclasses); + } + + + /** + * 取得所有的接口。 + * + * @return 所有的接口 + */ + public List getInterfaces() { + return Collections.unmodifiableList(interfaces); + } + } + + + /* + * ========================================================================== + * == + */ + /* 有关primitive类型的方法。 */ + /* + * ========================================================================== + * == + */ + + /** + * 返回指定类型所对应的primitive类型。 + * + * @param clazz + * 要检查的类型 + * + * @return 如果指定类型为null或不是primitive类型的包装类，则返回null + * ，否则返回相应的primitive类型。 + */ + public static Class getPrimitiveType(Class clazz) { + if (clazz == null) { + return null; + } + + if (clazz.isPrimitive()) { + return clazz; + } + + if (clazz.equals(Long.class)) { + return long.class; + } + + if (clazz.equals(Integer.class)) { + return int.class; + } + + if (clazz.equals(Short.class)) { + return short.class; + } + + if (clazz.equals(Byte.class)) { + return byte.class; + } + + if (clazz.equals(Double.class)) { + return double.class; + } + + if (clazz.equals(Float.class)) { + return float.class; + } + + if (clazz.equals(Boolean.class)) { + return boolean.class; + } + + if (clazz.equals(Character.class)) { + return char.class; + } + + return null; + } + + + /** + * 返回指定类型所对应的非primitive类型。 + * + * @param clazz + * 要检查的类型 + * + * @return 如果指定类型为null，则返回null + * ，如果是primitive类型，则返回相应的包装类，否则返回原始的类型。 + */ + public static Class getNonPrimitiveType(Class clazz) { + if (clazz == null) { + return null; + } + + if (!clazz.isPrimitive()) { + return clazz; + } + + if (clazz.equals(long.class)) { + return Long.class; + } + + if (clazz.equals(int.class)) { + return Integer.class; + } + + if (clazz.equals(short.class)) { + return Short.class; + } + + if (clazz.equals(byte.class)) { + return Byte.class; + } + + if (clazz.equals(double.class)) { + return Double.class; + } + + if (clazz.equals(float.class)) { + return Float.class; + } + + if (clazz.equals(boolean.class)) { + return Boolean.class; + } + + if (clazz.equals(char.class)) { + return Character.class; + } + + return null; + } +} diff --git a/rocketmq-filtersrv/src/main/java/com/alibaba/common/lang/CloneNotSupportedException.java b/rocketmq-filtersrv/src/main/java/com/alibaba/common/lang/CloneNotSupportedException.java new file mode 100644 index 000000000..50054c673 --- /dev/null +++ b/rocketmq-filtersrv/src/main/java/com/alibaba/common/lang/CloneNotSupportedException.java @@ -0,0 +1,63 @@ +package com.alibaba.common.lang; + +import com.alibaba.common.lang.exception.ChainedRuntimeException; + + +/** + * 当ObjectUtil.clone方法被调用时，如果被复制的对象不支持该操作，则抛出该异常。 + * + *

+ * 注意，和java.lang.CloneNotSupportedException不同，该异常从 + * RuntimeException派生。 + *

+ * + * @author Michael Zhou + * @version $Id: CloneNotSupportedException.java 1291 2005-03-04 03:23:30Z + * baobao $ + */ +public class CloneNotSupportedException extends ChainedRuntimeException { + private static final long serialVersionUID = 3257281439807584562L; + + + /** + * 构造一个空的异常. + */ + public CloneNotSupportedException() { + super(); + } + + + /** + * 构造一个异常, 指明异常的详细信息. + * + * @param message + * 详细信息 + */ + public CloneNotSupportedException(String message) { + super(message); + } + + + /** + * 构造一个异常, 指明引起这个异常的起因. + * + * @param cause + * 异常的起因 + */ + public CloneNotSupportedException(Throwable cause) { + super(cause); + } + + + /** + * 构造一个异常, 指明引起这个异常的起因. + * + * @param message + * 详细信息 + * @param cause + * 异常的起因 + */ + public CloneNotSupportedException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/rocketmq-filtersrv/src/main/java/com/alibaba/common/lang/ExceptionUtil.java b/rocketmq-filtersrv/src/main/java/com/alibaba/common/lang/ExceptionUtil.java new file mode 100644 index 000000000..bc97a5ffc --- /dev/null +++ b/rocketmq-filtersrv/src/main/java/com/alibaba/common/lang/ExceptionUtil.java @@ -0,0 +1,31 @@ +package com.alibaba.common.lang; + +import java.io.PrintWriter; +import java.io.StringWriter; + + +/** + * 处理异常的工具类。 + * + * @author Michael Zhou + * @version $Id: ExceptionUtil.java 965 2004-04-28 03:20:05Z baobao $ + */ +public class ExceptionUtil { + /** + * 取得异常的stacktrace字符串。 + * + * @param throwable + * 异常 + * + * @return stacktrace字符串 + */ + public static String getStackTrace(Throwable throwable) { + StringWriter buffer = new StringWriter(); + PrintWriter out = new PrintWriter(buffer); + + throwable.printStackTrace(out); + out.flush(); + + return buffer.toString(); + } +} diff --git a/rocketmq-filtersrv/src/main/java/com/alibaba/common/lang/ObjectUtil.java b/rocketmq-filtersrv/src/main/java/com/alibaba/common/lang/ObjectUtil.java new file mode 100644 index 000000000..f72216fa9 --- /dev/null +++ b/rocketmq-filtersrv/src/main/java/com/alibaba/common/lang/ObjectUtil.java @@ -0,0 +1,457 @@ +package com.alibaba.common.lang; + +import java.io.Serializable; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; + + +/** + * 有关Object处理的工具类。 + * + *

+ * 这个类中的每个方法都可以“安全”地处理null，而不会抛出NullPointerException。 + *

+ * + * @author Michael Zhou + * @version $Id: ObjectUtil.java 509 2004-02-16 05:42:07Z baobao $ + */ +public class ObjectUtil { + /* + * ========================================================================== + * == + */ + /* 常量和singleton。 */ + /* + * ========================================================================== + * == + */ + + /** + * 用于表示null的常量。 + * + *

+ * 例如，HashMap.get(Object)方法返回null有两种可能： 值不存在或值为 + * null。而这个singleton可用来区别这两种情形。 + *

+ * + *

+ * 另一个例子是，Hashtable的值不能为null。 + *

+ */ + public static final Object NULL = new Serializable() { + private static final long serialVersionUID = 7092611880189329093L; + + + private Object readResolve() { + return NULL; + } + }; + + + /* + * ========================================================================== + * == + */ + /* 默认值函数。 */ + /* */ + /* 当对象为null时，将对象转换成指定的默认对象。 */ + /* + * ========================================================================== + * == + */ + + /** + * 如果对象为null，则返回指定默认对象，否则返回对象本身。 + * + * 
+     * ObjectUtil.defaultIfNull(null, null)      = null
+     * ObjectUtil.defaultIfNull(null, "")        = ""
+     * ObjectUtil.defaultIfNull(null, "zz")      = "zz"
+     * ObjectUtil.defaultIfNull("abc", *)        = "abc"
+     * ObjectUtil.defaultIfNull(Boolean.TRUE, *) = Boolean.TRUE
+     *
+ * + * @param object + * 要测试的对象 + * @param defaultValue + * 默认值 + * + * @return 对象本身或默认对象 + */ + public static Object defaultIfNull(Object object, Object defaultValue) { + return (object != null) ? object : defaultValue; + } + + + /* + * ========================================================================== + * == + */ + /* 比较函数。 */ + /* */ + /* 以下方法用来比较两个对象是否相同。 */ + /* + * ========================================================================== + * == + */ + + /** + * 比较两个对象是否完全相等。 + * + *

+ * 此方法可以正确地比较多维数组。 + * + * 

+     * ObjectUtil.equals(null, null)                  = true
+     * ObjectUtil.equals(null, "")                    = false
+     * ObjectUtil.equals("", null)                    = false
+     * ObjectUtil.equals("", "")                      = true
+     * ObjectUtil.equals(Boolean.TRUE, null)          = false
+     * ObjectUtil.equals(Boolean.TRUE, "true")        = false
+     * ObjectUtil.equals(Boolean.TRUE, Boolean.TRUE)  = true
+     * ObjectUtil.equals(Boolean.TRUE, Boolean.FALSE) = false
+     *
+ * + *

+ * + * @param object1 + * 对象1 + * @param object2 + * 对象2 + * + * @return 如果相等, 则返回true + */ + public static boolean equals(Object object1, Object object2) { + return ArrayUtil.equals(object1, object2); + } + + + /* + * ========================================================================== + * == + */ + /* Hashcode函数。 */ + /* */ + /* 以下方法用来取得对象的hash code。 */ + /* + * ========================================================================== + * == + */ + + /** + * 取得对象的hash值, 如果对象为null, 则返回0。 + * + *

+ * 此方法可以正确地处理多维数组。 + *

+ * + * @param object + * 对象 + * + * @return hash值 + */ + public static int hashCode(Object object) { + return ArrayUtil.hashCode(object); + } + + + /** + * 取得对象的原始的hash值, 如果对象为null, 则返回0。 + * + *

+ * 该方法使用System.identityHashCode来取得hash值，该值不受对象本身的 + * hashCode方法的影响。 + *

+ * + * @param object + * 对象 + * + * @return hash值 + */ + public static int identityHashCode(Object object) { + return (object == null) ? 0 : System.identityHashCode(object); + } + + + /* + * ========================================================================== + * == + */ + /* 取得对象的identity。 */ + /* + * ========================================================================== + * == + */ + + /** + * 取得对象自身的identity，如同对象没有覆盖toString()方法时， + * Object.toString()的原始输出。 + * + * 
+     * ObjectUtil.identityToString(null)          = null
+     * ObjectUtil.identityToString("")            = "java.lang.String@1e23"
+     * ObjectUtil.identityToString(Boolean.TRUE)  = "java.lang.Boolean@7fa"
+     * ObjectUtil.identityToString(new int[0])    = "int[]@7fa"
+     * ObjectUtil.identityToString(new Object[0]) = "java.lang.Object[]@7fa"
+     *
+ * + * @param object + * 对象 + * + * @return 对象的identity，如果对象是null，则返回null + */ + public static String identityToString(Object object) { + if (object == null) { + return null; + } + + return appendIdentityToString(null, object).toString(); + } + + + /** + * 取得对象自身的identity，如同对象没有覆盖toString()方法时， + * Object.toString()的原始输出。 + * + * 
+     * ObjectUtil.identityToString(null, "NULL")            = "NULL"
+     * ObjectUtil.identityToString("", "NULL")              = "java.lang.String@1e23"
+     * ObjectUtil.identityToString(Boolean.TRUE, "NULL")    = "java.lang.Boolean@7fa"
+     * ObjectUtil.identityToString(new int[0], "NULL")      = "int[]@7fa"
+     * ObjectUtil.identityToString(new Object[0], "NULL")   = "java.lang.Object[]@7fa"
+     *
+ * + * @param object + * 对象 + * @param nullStr + * 如果对象为null，则返回该字符串 + * + * @return 对象的identity，如果对象是null，则返回指定字符串 + */ + public static String identityToString(Object object, String nullStr) { + if (object == null) { + return nullStr; + } + + return appendIdentityToString(null, object).toString(); + } + + + /** + * 将对象自身的identity——如同对象没有覆盖toString()方法时， + * Object.toString()的原始输出——追加到StringBuffer中。 + * + * 
+     * ObjectUtil.appendIdentityToString(*, null)            = null
+     * ObjectUtil.appendIdentityToString(null, "")           = "java.lang.String@1e23"
+     * ObjectUtil.appendIdentityToString(null, Boolean.TRUE) = "java.lang.Boolean@7fa"
+     * ObjectUtil.appendIdentityToString(buf, Boolean.TRUE)  = buf.append("java.lang.Boolean@7fa")
+     * ObjectUtil.appendIdentityToString(buf, new int[0])    = buf.append("int[]@7fa")
+     * ObjectUtil.appendIdentityToString(buf, new Object[0]) = buf.append("java.lang.Object[]@7fa")
+     *
+ * + * @param buffer + * StringBuffer对象，如果是null，则创建新的 + * @param object + * 对象 + * + * @return StringBuffer对象，如果对象为null，则返回 + * null + */ + public static StringBuffer appendIdentityToString(StringBuffer buffer, Object object) { + if (object == null) { + return null; + } + + if (buffer == null) { + buffer = new StringBuffer(); + } + + buffer.append(ClassUtil.getClassNameForObject(object)); + + return buffer.append('@').append(Integer.toHexString(identityHashCode(object))); + } + + + /* + * ========================================================================== + * == + */ + /* Clone函数。 */ + /* */ + /* 以下方法调用Object.clone方法，默认是“浅复制”（shallow copy）。 */ + /* + * ========================================================================== + * == + */ + + /** + * 复制一个对象。如果对象为null，则返回null。 + * + *

+ * 此方法调用Object.clone方法，默认只进行“浅复制”。 对于数组，调用 + * ArrayUtil.clone方法更高效。 + *

+ * + * @param array + * 要复制的数组 + * + * @return 数组的复本，如果原始数组为null，则返回null + */ + public static Object clone(Object array) { + if (array == null) { + return null; + } + + // 对数组特殊处理 + if (array instanceof Object[]) { + return ArrayUtil.clone((Object[]) array); + } + + if (array instanceof long[]) { + return ArrayUtil.clone((long[]) array); + } + + if (array instanceof int[]) { + return ArrayUtil.clone((int[]) array); + } + + if (array instanceof short[]) { + return ArrayUtil.clone((short[]) array); + } + + if (array instanceof byte[]) { + return ArrayUtil.clone((byte[]) array); + } + + if (array instanceof double[]) { + return ArrayUtil.clone((double[]) array); + } + + if (array instanceof float[]) { + return ArrayUtil.clone((float[]) array); + } + + if (array instanceof boolean[]) { + return ArrayUtil.clone((boolean[]) array); + } + + if (array instanceof char[]) { + return ArrayUtil.clone((char[]) array); + } + + // Not cloneable + if (!(array instanceof Cloneable)) { + throw new CloneNotSupportedException("Object of class " + array.getClass().getName() + + " is not Cloneable"); + } + + // 用reflection调用clone方法 + Class clazz = array.getClass(); + + try { + Method cloneMethod = clazz.getMethod("clone", ArrayUtil.EMPTY_CLASS_ARRAY); + + return cloneMethod.invoke(array, ArrayUtil.EMPTY_OBJECT_ARRAY); + } + catch (NoSuchMethodException e) { + throw new CloneNotSupportedException(e); + } + catch (IllegalArgumentException e) { + throw new CloneNotSupportedException(e); + } + catch (IllegalAccessException e) { + throw new CloneNotSupportedException(e); + } + catch (InvocationTargetException e) { + throw new CloneNotSupportedException(e); + } + } + + + /* + * ========================================================================== + * == + */ + /* 比较对象的类型。 */ + /* + * ========================================================================== + * == + */ + + /** + * 检查两个对象是否属于相同类型。null将被看作任意类型。 + * + * @param object1 + * 对象1 + * @param object2 + * 对象2 + * + * @return 如果两个对象有相同的类型，则返回true + */ + public static boolean isSameType(Object object1, Object object2) { + if ((object1 == null) || (object2 == null)) { + return true; + } + + return object1.getClass().equals(object2.getClass()); + } + + + /* + * ========================================================================== + * == + */ + /* toString方法。 */ + /* + * ========================================================================== + * == + */ + + /** + * 取得对象的toString()的值，如果对象为null，则返回空字符串 + * ""。 + * + * 
+     * ObjectUtil.toString(null)         = ""
+     * ObjectUtil.toString("")           = ""
+     * ObjectUtil.toString("bat")        = "bat"
+     * ObjectUtil.toString(Boolean.TRUE) = "true"
+     * ObjectUtil.toString([1, 2, 3])    = "[1, 2, 3]"
+     *
+ * + * @param object + * 对象 + * + * @return 对象的toString()的返回值，或空字符串"" + */ + public static String toString(Object object) { + return (object == null) ? StringUtil.EMPTY_STRING : (object.getClass().isArray() ? ArrayUtil + .toString(object) : object.toString()); + } + + + /** + * 取得对象的toString()的值，如果对象为null，则返回指定字符串。 + * + * 
+     * ObjectUtil.toString(null, null)           = null
+     * ObjectUtil.toString(null, "null")         = "null"
+     * ObjectUtil.toString("", "null")           = ""
+     * ObjectUtil.toString("bat", "null")        = "bat"
+     * ObjectUtil.toString(Boolean.TRUE, "null") = "true"
+     * ObjectUtil.toString([1, 2, 3], "null")    = "[1, 2, 3]"
+     *
+ * + * @param object + * 对象 + * @param nullStr + * 如果对象为null，则返回该字符串 + * + * @return 对象的toString()的返回值，或指定字符串 + */ + public static String toString(Object object, String nullStr) { + return (object == null) ? nullStr : (object.getClass().isArray() ? ArrayUtil.toString(object) + : object.toString()); + } +} diff --git a/rocketmq-filtersrv/src/main/java/com/alibaba/common/lang/ServiceNotFoundException.java b/rocketmq-filtersrv/src/main/java/com/alibaba/common/lang/ServiceNotFoundException.java new file mode 100644 index 000000000..9a40fbbba --- /dev/null +++ b/rocketmq-filtersrv/src/main/java/com/alibaba/common/lang/ServiceNotFoundException.java @@ -0,0 +1,117 @@ +package com.alibaba.common.lang; + +import java.io.PrintStream; +import java.io.PrintWriter; + +import com.alibaba.common.lang.exception.ChainedThrowable; +import com.alibaba.common.lang.exception.ChainedThrowableDelegate; + + +/** + * 代表META-INF/services/中的文件未找到或读文件失败的异常。 + * + * @author Michael Zhou + * @version $Id: ServiceNotFoundException.java 1291 2005-03-04 03:23:30Z baobao + * $ + */ +public class ServiceNotFoundException extends ClassNotFoundException implements ChainedThrowable { + private static final long serialVersionUID = 3258126964232566584L; + private final ChainedThrowable delegate = new ChainedThrowableDelegate(this); + private Throwable cause; + + + /** + * 构造一个空的异常. + */ + public ServiceNotFoundException() { + super(); + } + + + /** + * 构造一个异常, 指明异常的详细信息. + * + * @param message + * 详细信息 + */ + public ServiceNotFoundException(String message) { + super(message); + } + + + /** + * 构造一个异常, 指明引起这个异常的起因. + * + * @param cause + * 异常的起因 + */ + public ServiceNotFoundException(Throwable cause) { + super((cause == null) ? null : cause.getMessage()); + this.cause = cause; + } + + + /** + * 构造一个异常, 指明引起这个异常的起因. + * + * @param message + * 详细信息 + * @param cause + * 异常的起因 + */ + public ServiceNotFoundException(String message, Throwable cause) { + super(message); + this.cause = cause; + } + + + /** + * 取得引起这个异常的起因. + * + * @return 异常的起因. + */ + public Throwable getCause() { + return cause; + } + + + /** + * 打印调用栈到标准错误. + */ + public void printStackTrace() { + delegate.printStackTrace(); + } + + + /** + * 打印调用栈到指定输出流. + * + * @param stream + * 输出字节流. + */ + public void printStackTrace(PrintStream stream) { + delegate.printStackTrace(stream); + } + + + /** + * 打印调用栈到指定输出流. + * + * @param writer + * 输出字符流. + */ + public void printStackTrace(PrintWriter writer) { + delegate.printStackTrace(writer); + } + + + /** + * 打印异常的调用栈, 不包括起因异常的信息. + * + * @param writer + * 打印到输出流 + */ + public void printCurrentStackTrace(PrintWriter writer) { + super.printStackTrace(writer); + } +} diff --git a/rocketmq-filtersrv/src/main/java/com/alibaba/common/lang/StringUtil.java b/rocketmq-filtersrv/src/main/java/com/alibaba/common/lang/StringUtil.java new file mode 100644 index 000000000..ef89a2766 --- /dev/null +++ b/rocketmq-filtersrv/src/main/java/com/alibaba/common/lang/StringUtil.java @@ -0,0 +1,4949 @@ +package com.alibaba.common.lang; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + + +/** + * 有关字符串处理的工具类。 + * + *

+ * 这个类中的每个方法都可以“安全”地处理null，而不会抛出NullPointerException。 + *

+ * + * @author Michael Zhou + * @version $Id: StringUtil.java 1149 2004-08-10 02:01:41Z baobao $ + */ +public class StringUtil { + /* + * ========================================================================== + * == + */ + /* 常量和singleton。 */ + /* + * ========================================================================== + * == + */ + + /** 空字符串。 */ + public static final String EMPTY_STRING = ""; + + + /* + * ========================================================================== + * == + */ + /* 判空函数。 */ + /* */ + /* 以下方法用来判定一个字符串是否为： */ + /* 1. null */ + /* 2. empty - "" */ + /* 3. blank - "全部是空白" - 空白由Character.isWhitespace所定义。 */ + /* + * ========================================================================== + * == + */ + + /** + * 检查字符串是否为null或空字符串""。 + * + * 
+     * StringUtil.isEmpty(null)      = true
+     * StringUtil.isEmpty("")        = true
+     * StringUtil.isEmpty(" ")       = false
+     * StringUtil.isEmpty("bob")     = false
+     * StringUtil.isEmpty("  bob  ") = false
+     *
+ * + * @param str + * 要检查的字符串 + * + * @return 如果为空, 则返回true + */ + public static boolean isEmpty(String str) { + return ((str == null) || (str.length() == 0)); + } + + + /** + * 检查字符串是否不是null和空字符串""。 + * + * 
+     * StringUtil.isEmpty(null)      = false
+     * StringUtil.isEmpty("")        = false
+     * StringUtil.isEmpty(" ")       = true
+     * StringUtil.isEmpty("bob")     = true
+     * StringUtil.isEmpty("  bob  ") = true
+     *
+ * + * @param str + * 要检查的字符串 + * + * @return 如果不为空, 则返回true + */ + public static boolean isNotEmpty(String str) { + return ((str != null) && (str.length() > 0)); + } + + + /** + * 检查字符串是否是空白：null、空字符串""或只有空白字符。 + * + * 
+     * StringUtil.isBlank(null)      = true
+     * StringUtil.isBlank("")        = true
+     * StringUtil.isBlank(" ")       = true
+     * StringUtil.isBlank("bob")     = false
+     * StringUtil.isBlank("  bob  ") = false
+     *
+ * + * @param str + * 要检查的字符串 + * + * @return 如果为空白, 则返回true + */ + public static boolean isBlank(String str) { + int length; + + if ((str == null) || ((length = str.length()) == 0)) { + return true; + } + + for (int i = 0; i < length; i++) { + if (!Character.isWhitespace(str.charAt(i))) { + return false; + } + } + + return true; + } + + + /** + * 检查字符串是否不是空白：null、空字符串""或只有空白字符。 + * + * 
+     * StringUtil.isBlank(null)      = false
+     * StringUtil.isBlank("")        = false
+     * StringUtil.isBlank(" ")       = false
+     * StringUtil.isBlank("bob")     = true
+     * StringUtil.isBlank("  bob  ") = true
+     *
+ * + * @param str + * 要检查的字符串 + * + * @return 如果为空白, 则返回true + */ + public static boolean isNotBlank(String str) { + int length; + + if ((str == null) || ((length = str.length()) == 0)) { + return false; + } + + for (int i = 0; i < length; i++) { + if (!Character.isWhitespace(str.charAt(i))) { + return true; + } + } + + return false; + } + + + /* + * ========================================================================== + * == + */ + /* 默认值函数。 */ + /* */ + /* 当字符串为null、empty或blank时，将字符串转换成指定的默认字符串。 */ + /* + * ========================================================================== + * == + */ + + /** + * 如果字符串是null，则返回空字符串""，否则返回字符串本身。 + * + * 
+     * StringUtil.defaultIfNull(null)  = ""
+     * StringUtil.defaultIfNull("")    = ""
+     * StringUtil.defaultIfNull("  ")  = "  "
+     * StringUtil.defaultIfNull("bat") = "bat"
+     *
+ * + * @param str + * 要转换的字符串 + * + * @return 字符串本身或空字符串"" + */ + public static String defaultIfNull(String str) { + return (str == null) ? EMPTY_STRING : str; + } + + + /** + * 如果字符串是null，则返回指定默认字符串，否则返回字符串本身。 + * + * 
+     * StringUtil.defaultIfNull(null, "default")  = "default"
+     * StringUtil.defaultIfNull("", "default")    = ""
+     * StringUtil.defaultIfNull("  ", "default")  = "  "
+     * StringUtil.defaultIfNull("bat", "default") = "bat"
+     *
+ * + * @param str + * 要转换的字符串 + * @param defaultStr + * 默认字符串 + * + * @return 字符串本身或指定的默认字符串 + */ + public static String defaultIfNull(String str, String defaultStr) { + return (str == null) ? defaultStr : str; + } + + + /** + * 如果字符串是null或空字符串""，则返回空字符串"" + * ，否则返回字符串本身。 + * + *

+ * 此方法实际上和defaultIfNull(String)等效。 + * + * 

+     * StringUtil.defaultIfEmpty(null)  = ""
+     * StringUtil.defaultIfEmpty("")    = ""
+     * StringUtil.defaultIfEmpty("  ")  = "  "
+     * StringUtil.defaultIfEmpty("bat") = "bat"
+     *
+ * + *

+ * + * @param str + * 要转换的字符串 + * + * @return 字符串本身或空字符串"" + */ + public static String defaultIfEmpty(String str) { + return (str == null) ? EMPTY_STRING : str; + } + + + /** + * 如果字符串是null或空字符串""，则返回指定默认字符串，否则返回字符串本身。 + * + * 
+     * StringUtil.defaultIfEmpty(null, "default")  = "default"
+     * StringUtil.defaultIfEmpty("", "default")    = "default"
+     * StringUtil.defaultIfEmpty("  ", "default")  = "  "
+     * StringUtil.defaultIfEmpty("bat", "default") = "bat"
+     *
+ * + * @param str + * 要转换的字符串 + * @param defaultStr + * 默认字符串 + * + * @return 字符串本身或指定的默认字符串 + */ + public static String defaultIfEmpty(String str, String defaultStr) { + return ((str == null) || (str.length() == 0)) ? defaultStr : str; + } + + + /** + * 如果字符串是空白：null、空字符串""或只有空白字符，则返回空字符串 + * ""，否则返回字符串本身。 + * + * 
+     * StringUtil.defaultIfBlank(null)  = ""
+     * StringUtil.defaultIfBlank("")    = ""
+     * StringUtil.defaultIfBlank("  ")  = ""
+     * StringUtil.defaultIfBlank("bat") = "bat"
+     *
+ * + * @param str + * 要转换的字符串 + * + * @return 字符串本身或空字符串"" + */ + public static String defaultIfBlank(String str) { + return isBlank(str) ? EMPTY_STRING : str; + } + + + /** + * 如果字符串是null或空字符串""，则返回指定默认字符串，否则返回字符串本身。 + * + * 
+     * StringUtil.defaultIfBlank(null, "default")  = "default"
+     * StringUtil.defaultIfBlank("", "default")    = "default"
+     * StringUtil.defaultIfBlank("  ", "default")  = "default"
+     * StringUtil.defaultIfBlank("bat", "default") = "bat"
+     *
+ * + * @param str + * 要转换的字符串 + * @param defaultStr + * 默认字符串 + * + * @return 字符串本身或指定的默认字符串 + */ + public static String defaultIfBlank(String str, String defaultStr) { + return isBlank(str) ? defaultStr : str; + } + + + /* + * ========================================================================== + * == + */ + /* 去空白（或指定字符）的函数。 */ + /* */ + /* 以下方法用来除去一个字串中的空白或指定字符。 */ + /* + * ========================================================================== + * == + */ + + /** + * 除去字符串头尾部的空白，如果字符串是null，依然返回null。 + * + *

+ * 注意，和String.trim不同，此方法使用Character.isWhitespace + * 来判定空白， 因而可以除去英文字符集之外的其它空白，如中文空格。 + * + * 

+     * StringUtil.trim(null)          = null
+     * StringUtil.trim("")            = ""
+     * StringUtil.trim("     ")       = ""
+     * StringUtil.trim("abc")         = "abc"
+     * StringUtil.trim("    abc    ") = "abc"
+     *
+ * + *

+ * + * @param str + * 要处理的字符串 + * + * @return 除去空白的字符串，如果原字串为null，则返回null + */ + public static String trim(String str) { + return trim(str, null, 0); + } + + + /** + * 除去字符串头尾部的指定字符，如果字符串是null，依然返回null。 + * + * 
+     * StringUtil.trim(null, *)          = null
+     * StringUtil.trim("", *)            = ""
+     * StringUtil.trim("abc", null)      = "abc"
+     * StringUtil.trim("  abc", null)    = "abc"
+     * StringUtil.trim("abc  ", null)    = "abc"
+     * StringUtil.trim(" abc ", null)    = "abc"
+     * StringUtil.trim("  abcyx", "xyz") = "  abc"
+     *
+ * + * @param str + * 要处理的字符串 + * @param stripChars + * 要除去的字符，如果为null表示除去空白字符 + * + * @return 除去指定字符后的的字符串，如果原字串为null，则返回null + */ + public static String trim(String str, String stripChars) { + return trim(str, stripChars, 0); + } + + + /** + * 除去字符串头部的空白，如果字符串是null，则返回null。 + * + *

+ * 注意，和String.trim不同，此方法使用Character.isWhitespace + * 来判定空白， 因而可以除去英文字符集之外的其它空白，如中文空格。 + * + * 

+     * StringUtil.trimStart(null)         = null
+     * StringUtil.trimStart("")           = ""
+     * StringUtil.trimStart("abc")        = "abc"
+     * StringUtil.trimStart("  abc")      = "abc"
+     * StringUtil.trimStart("abc  ")      = "abc  "
+     * StringUtil.trimStart(" abc ")      = "abc "
+     *
+ * + *

+ * + * @param str + * 要处理的字符串 + * + * @return 除去空白的字符串，如果原字串为null或结果字符串为""，则返回 + * null + */ + public static String trimStart(String str) { + return trim(str, null, -1); + } + + + /** + * 除去字符串头部的指定字符，如果字符串是null，依然返回null。 + * + * 
+     * StringUtil.trimStart(null, *)          = null
+     * StringUtil.trimStart("", *)            = ""
+     * StringUtil.trimStart("abc", "")        = "abc"
+     * StringUtil.trimStart("abc", null)      = "abc"
+     * StringUtil.trimStart("  abc", null)    = "abc"
+     * StringUtil.trimStart("abc  ", null)    = "abc  "
+     * StringUtil.trimStart(" abc ", null)    = "abc "
+     * StringUtil.trimStart("yxabc  ", "xyz") = "abc  "
+     *
+ * + * @param str + * 要处理的字符串 + * @param stripChars + * 要除去的字符，如果为null表示除去空白字符 + * + * @return 除去指定字符后的的字符串，如果原字串为null，则返回null + */ + public static String trimStart(String str, String stripChars) { + return trim(str, stripChars, -1); + } + + + /** + * 除去字符串尾部的空白，如果字符串是null，则返回null。 + * + *

+ * 注意，和String.trim不同，此方法使用Character.isWhitespace + * 来判定空白， 因而可以除去英文字符集之外的其它空白，如中文空格。 + * + * 

+     * StringUtil.trimEnd(null)       = null
+     * StringUtil.trimEnd("")         = ""
+     * StringUtil.trimEnd("abc")      = "abc"
+     * StringUtil.trimEnd("  abc")    = "  abc"
+     * StringUtil.trimEnd("abc  ")    = "abc"
+     * StringUtil.trimEnd(" abc ")    = " abc"
+     *
+ * + *

+ * + * @param str + * 要处理的字符串 + * + * @return 除去空白的字符串，如果原字串为null或结果字符串为""，则返回 + * null + */ + public static String trimEnd(String str) { + return trim(str, null, 1); + } + + + /** + * 除去字符串尾部的指定字符，如果字符串是null，依然返回null。 + * + * 
+     * StringUtil.trimEnd(null, *)          = null
+     * StringUtil.trimEnd("", *)            = ""
+     * StringUtil.trimEnd("abc", "")        = "abc"
+     * StringUtil.trimEnd("abc", null)      = "abc"
+     * StringUtil.trimEnd("  abc", null)    = "  abc"
+     * StringUtil.trimEnd("abc  ", null)    = "abc"
+     * StringUtil.trimEnd(" abc ", null)    = " abc"
+     * StringUtil.trimEnd("  abcyx", "xyz") = "  abc"
+     *
+ * + * @param str + * 要处理的字符串 + * @param stripChars + * 要除去的字符，如果为null表示除去空白字符 + * + * @return 除去指定字符后的的字符串，如果原字串为null，则返回null + */ + public static String trimEnd(String str, String stripChars) { + return trim(str, stripChars, 1); + } + + + /** + * 除去字符串头尾部的空白，如果结果字符串是空字符串""，则返回null。 + * + *

+ * 注意，和String.trim不同，此方法使用Character.isWhitespace + * 来判定空白， 因而可以除去英文字符集之外的其它空白，如中文空格。 + * + * 

+     * StringUtil.trimToNull(null)          = null
+     * StringUtil.trimToNull("")            = null
+     * StringUtil.trimToNull("     ")       = null
+     * StringUtil.trimToNull("abc")         = "abc"
+     * StringUtil.trimToNull("    abc    ") = "abc"
+     *
+ * + *

+ * + * @param str + * 要处理的字符串 + * + * @return 除去空白的字符串，如果原字串为null或结果字符串为""，则返回 + * null + */ + public static String trimToNull(String str) { + return trimToNull(str, null); + } + + + /** + * 除去字符串头尾部的空白，如果结果字符串是空字符串""，则返回null。 + * + *

+ * 注意，和String.trim不同，此方法使用Character.isWhitespace + * 来判定空白， 因而可以除去英文字符集之外的其它空白，如中文空格。 + * + * 

+     * StringUtil.trim(null, *)          = null
+     * StringUtil.trim("", *)            = null
+     * StringUtil.trim("abc", null)      = "abc"
+     * StringUtil.trim("  abc", null)    = "abc"
+     * StringUtil.trim("abc  ", null)    = "abc"
+     * StringUtil.trim(" abc ", null)    = "abc"
+     * StringUtil.trim("  abcyx", "xyz") = "  abc"
+     *
+ * + *

+ * + * @param str + * 要处理的字符串 + * @param stripChars + * 要除去的字符，如果为null表示除去空白字符 + * + * @return 除去空白的字符串，如果原字串为null或结果字符串为""，则返回 + * null + */ + public static String trimToNull(String str, String stripChars) { + String result = trim(str, stripChars); + + if ((result == null) || (result.length() == 0)) { + return null; + } + + return result; + } + + + /** + * 除去字符串头尾部的空白，如果字符串是null，则返回空字符串""。 + * + *

+ * 注意，和String.trim不同，此方法使用Character.isWhitespace + * 来判定空白， 因而可以除去英文字符集之外的其它空白，如中文空格。 + * + * 

+     * StringUtil.trimToEmpty(null)          = ""
+     * StringUtil.trimToEmpty("")            = ""
+     * StringUtil.trimToEmpty("     ")       = ""
+     * StringUtil.trimToEmpty("abc")         = "abc"
+     * StringUtil.trimToEmpty("    abc    ") = "abc"
+     *
+ * + *

+ * + * @param str + * 要处理的字符串 + * + * @return 除去空白的字符串，如果原字串为null或结果字符串为""，则返回 + * null + */ + public static String trimToEmpty(String str) { + return trimToEmpty(str, null); + } + + + /** + * 除去字符串头尾部的空白，如果字符串是null，则返回空字符串""。 + * + *

+ * 注意，和String.trim不同，此方法使用Character.isWhitespace + * 来判定空白， 因而可以除去英文字符集之外的其它空白，如中文空格。 + * + * 

+     * StringUtil.trim(null, *)          = ""
+     * StringUtil.trim("", *)            = ""
+     * StringUtil.trim("abc", null)      = "abc"
+     * StringUtil.trim("  abc", null)    = "abc"
+     * StringUtil.trim("abc  ", null)    = "abc"
+     * StringUtil.trim(" abc ", null)    = "abc"
+     * StringUtil.trim("  abcyx", "xyz") = "  abc"
+     *
+ * + *

+ * + * @param str + * 要处理的字符串 + * + * @return 除去空白的字符串，如果原字串为null或结果字符串为""，则返回 + * null + */ + public static String trimToEmpty(String str, String stripChars) { + String result = trim(str, stripChars); + + if (result == null) { + return EMPTY_STRING; + } + + return result; + } + + + /** + * 除去字符串头尾部的指定字符，如果字符串是null，依然返回null。 + * + * 
+     * StringUtil.trim(null, *)          = null
+     * StringUtil.trim("", *)            = ""
+     * StringUtil.trim("abc", null)      = "abc"
+     * StringUtil.trim("  abc", null)    = "abc"
+     * StringUtil.trim("abc  ", null)    = "abc"
+     * StringUtil.trim(" abc ", null)    = "abc"
+     * StringUtil.trim("  abcyx", "xyz") = "  abc"
+     *
+ * + * @param str + * 要处理的字符串 + * @param stripChars + * 要除去的字符，如果为null表示除去空白字符 + * @param mode + * -1表示trimStart，0表示trim全部， + * 1表示trimEnd + * + * @return 除去指定字符后的的字符串，如果原字串为null，则返回null + */ + private static String trim(String str, String stripChars, int mode) { + if (str == null) { + return null; + } + + int length = str.length(); + int start = 0; + int end = length; + + // 扫描字符串头部 + if (mode <= 0) { + if (stripChars == null) { + while ((start < end) && (Character.isWhitespace(str.charAt(start)))) { + start++; + } + } + else if (stripChars.length() == 0) { + return str; + } + else { + while ((start < end) && (stripChars.indexOf(str.charAt(start)) != -1)) { + start++; + } + } + } + + // 扫描字符串尾部 + if (mode >= 0) { + if (stripChars == null) { + while ((start < end) && (Character.isWhitespace(str.charAt(end - 1)))) { + end--; + } + } + else if (stripChars.length() == 0) { + return str; + } + else { + while ((start < end) && (stripChars.indexOf(str.charAt(end - 1)) != -1)) { + end--; + } + } + } + + if ((start > 0) || (end < length)) { + return str.substring(start, end); + } + + return str; + } + + + /* + * ========================================================================== + * == + */ + /* 比较函数。 */ + /* */ + /* 以下方法用来比较两个字符串是否相同。 */ + /* + * ========================================================================== + * == + */ + + /** + * 比较两个字符串（大小写敏感）。 + * + * 
+     * StringUtil.equals(null, null)   = true
+     * StringUtil.equals(null, "abc")  = false
+     * StringUtil.equals("abc", null)  = false
+     * StringUtil.equals("abc", "abc") = true
+     * StringUtil.equals("abc", "ABC") = false
+     *
+ * + * @param str1 + * 要比较的字符串1 + * @param str2 + * 要比较的字符串2 + * + * @return 如果两个字符串相同，或者都是null，则返回true + */ + public static boolean equals(String str1, String str2) { + if (str1 == null) { + return str2 == null; + } + + return str1.equals(str2); + } + + + /** + * 比较两个字符串（大小写不敏感）。 + * + * 
+     * StringUtil.equalsIgnoreCase(null, null)   = true
+     * StringUtil.equalsIgnoreCase(null, "abc")  = false
+     * StringUtil.equalsIgnoreCase("abc", null)  = false
+     * StringUtil.equalsIgnoreCase("abc", "abc") = true
+     * StringUtil.equalsIgnoreCase("abc", "ABC") = true
+     *
+ * + * @param str1 + * 要比较的字符串1 + * @param str2 + * 要比较的字符串2 + * + * @return 如果两个字符串相同，或者都是null，则返回true + */ + public static boolean equalsIgnoreCase(String str1, String str2) { + if (str1 == null) { + return str2 == null; + } + + return str1.equalsIgnoreCase(str2); + } + + + /* + * ========================================================================== + * == + */ + /* 字符串类型判定函数。 */ + /* */ + /* 判定字符串的类型是否为：字母、数字、空白等 */ + /* + * ========================================================================== + * == + */ + + /** + * 判断字符串是否只包含unicode字母。 + * + *

+ * null将返回false，空字符串""将返回 + * true。 + *

+ * + * 
+     * StringUtil.isAlpha(null)   = false
+     * StringUtil.isAlpha("")     = true
+     * StringUtil.isAlpha("  ")   = false
+     * StringUtil.isAlpha("abc")  = true
+     * StringUtil.isAlpha("ab2c") = false
+     * StringUtil.isAlpha("ab-c") = false
+     *
+ * + * @param str + * 要检查的字符串 + * + * @return 如果字符串非null并且全由unicode字母组成，则返回true + */ + public static boolean isAlpha(String str) { + if (str == null) { + return false; + } + + int length = str.length(); + + for (int i = 0; i < length; i++) { + if (!Character.isLetter(str.charAt(i))) { + return false; + } + } + + return true; + } + + + /** + * 判断字符串是否只包含unicode字母和空格' '。 + * + *

+ * null将返回false，空字符串""将返回 + * true。 + *

+ * + * 
+     * StringUtil.isAlphaSpace(null)   = false
+     * StringUtil.isAlphaSpace("")     = true
+     * StringUtil.isAlphaSpace("  ")   = true
+     * StringUtil.isAlphaSpace("abc")  = true
+     * StringUtil.isAlphaSpace("ab c") = true
+     * StringUtil.isAlphaSpace("ab2c") = false
+     * StringUtil.isAlphaSpace("ab-c") = false
+     *
+ * + * @param str + * 要检查的字符串 + * + * @return 如果字符串非null并且全由unicode字母和空格组成，则返回true + */ + public static boolean isAlphaSpace(String str) { + if (str == null) { + return false; + } + + int length = str.length(); + + for (int i = 0; i < length; i++) { + if (!Character.isLetter(str.charAt(i)) && (str.charAt(i) != ' ')) { + return false; + } + } + + return true; + } + + + /** + * 判断字符串是否只包含unicode字母和数字。 + * + *

+ * null将返回false，空字符串""将返回 + * true。 + *

+ * + * 
+     * StringUtil.isAlphanumeric(null)   = false
+     * StringUtil.isAlphanumeric("")     = true
+     * StringUtil.isAlphanumeric("  ")   = false
+     * StringUtil.isAlphanumeric("abc")  = true
+     * StringUtil.isAlphanumeric("ab c") = false
+     * StringUtil.isAlphanumeric("ab2c") = true
+     * StringUtil.isAlphanumeric("ab-c") = false
+     *
+ * + * @param str + * 要检查的字符串 + * + * @return 如果字符串非null并且全由unicode字母数字组成，则返回true + */ + public static boolean isAlphanumeric(String str) { + if (str == null) { + return false; + } + + int length = str.length(); + + for (int i = 0; i < length; i++) { + if (!Character.isLetterOrDigit(str.charAt(i))) { + return false; + } + } + + return true; + } + + + /** + * 判断字符串是否只包含unicode字母数字和空格' '。 + * + *

+ * null将返回false，空字符串""将返回 + * true。 + *

+ * + * 
+     * StringUtil.isAlphanumericSpace(null)   = false
+     * StringUtil.isAlphanumericSpace("")     = true
+     * StringUtil.isAlphanumericSpace("  ")   = true
+     * StringUtil.isAlphanumericSpace("abc")  = true
+     * StringUtil.isAlphanumericSpace("ab c") = true
+     * StringUtil.isAlphanumericSpace("ab2c") = true
+     * StringUtil.isAlphanumericSpace("ab-c") = false
+     *
+ * + * @param str + * 要检查的字符串 + * + * @return 如果字符串非null并且全由unicode字母数字和空格组成，则返回true + */ + public static boolean isAlphanumericSpace(String str) { + if (str == null) { + return false; + } + + int length = str.length(); + + for (int i = 0; i < length; i++) { + if (!Character.isLetterOrDigit(str.charAt(i)) && (str.charAt(i) != ' ')) { + return false; + } + } + + return true; + } + + + /** + * 判断字符串是否只包含unicode数字。 + * + *

+ * null将返回false，空字符串""将返回 + * true。 + *

+ * + * 
+     * StringUtil.isNumeric(null)   = false
+     * StringUtil.isNumeric("")     = true
+     * StringUtil.isNumeric("  ")   = false
+     * StringUtil.isNumeric("123")  = true
+     * StringUtil.isNumeric("12 3") = false
+     * StringUtil.isNumeric("ab2c") = false
+     * StringUtil.isNumeric("12-3") = false
+     * StringUtil.isNumeric("12.3") = false
+     *
+ * + * @param str + * 要检查的字符串 + * + * @return 如果字符串非null并且全由unicode数字组成，则返回true + */ + public static boolean isNumeric(String str) { + if (str == null) { + return false; + } + + int length = str.length(); + + for (int i = 0; i < length; i++) { + if (!Character.isDigit(str.charAt(i))) { + return false; + } + } + + return true; + } + + + /** + * 判断字符串是否只包含unicode数字和空格' '。 + * + *

+ * null将返回false，空字符串""将返回 + * true。 + *

+ * + * 
+     * StringUtil.isNumericSpace(null)   = false
+     * StringUtil.isNumericSpace("")     = true
+     * StringUtil.isNumericSpace("  ")   = true
+     * StringUtil.isNumericSpace("123")  = true
+     * StringUtil.isNumericSpace("12 3") = true
+     * StringUtil.isNumericSpace("ab2c") = false
+     * StringUtil.isNumericSpace("12-3") = false
+     * StringUtil.isNumericSpace("12.3") = false
+     *
+ * + * @param str + * 要检查的字符串 + * + * @return 如果字符串非null并且全由unicode数字和空格组成，则返回true + */ + public static boolean isNumericSpace(String str) { + if (str == null) { + return false; + } + + int length = str.length(); + + for (int i = 0; i < length; i++) { + if (!Character.isDigit(str.charAt(i)) && (str.charAt(i) != ' ')) { + return false; + } + } + + return true; + } + + + /** + * 判断字符串是否只包含unicode空白。 + * + *

+ * null将返回false，空字符串""将返回 + * true。 + *

+ * + * 
+     * StringUtil.isWhitespace(null)   = false
+     * StringUtil.isWhitespace("")     = true
+     * StringUtil.isWhitespace("  ")   = true
+     * StringUtil.isWhitespace("abc")  = false
+     * StringUtil.isWhitespace("ab2c") = false
+     * StringUtil.isWhitespace("ab-c") = false
+     *
+ * + * @param str + * 要检查的字符串 + * + * @return 如果字符串非null并且全由unicode空白组成，则返回true + */ + public static boolean isWhitespace(String str) { + if (str == null) { + return false; + } + + int length = str.length(); + + for (int i = 0; i < length; i++) { + if (!Character.isWhitespace(str.charAt(i))) { + return false; + } + } + + return true; + } + + + /* + * ========================================================================== + * == + */ + /* 大小写转换。 */ + /* + * ========================================================================== + * == + */ + + /** + * 将字符串转换成大写。 + * + *

+ * 如果字符串是null则返回null。 + * + * 

+     * StringUtil.toUpperCase(null)  = null
+     * StringUtil.toUpperCase("")    = ""
+     * StringUtil.toUpperCase("aBc") = "ABC"
+     *
+ * + *

+ * + * @param str + * 要转换的字符串 + * + * @return 大写字符串，如果原字符串为null，则返回null + */ + public static String toUpperCase(String str) { + if (str == null) { + return null; + } + + return str.toUpperCase(); + } + + + /** + * 将字符串转换成小写。 + * + *

+ * 如果字符串是null则返回null。 + * + * 

+     * StringUtil.toLowerCase(null)  = null
+     * StringUtil.toLowerCase("")    = ""
+     * StringUtil.toLowerCase("aBc") = "abc"
+     *
+ * + *

+ * + * @param str + * 要转换的字符串 + * + * @return 大写字符串，如果原字符串为null，则返回null + */ + public static String toLowerCase(String str) { + if (str == null) { + return null; + } + + return str.toLowerCase(); + } + + + /** + * 将字符串的首字符转成大写（Character.toTitleCase），其它字符不变。 + * + *

+ * 如果字符串是null则返回null。 + * + * 

+     * StringUtil.capitalize(null)  = null
+     * StringUtil.capitalize("")    = ""
+     * StringUtil.capitalize("cat") = "Cat"
+     * StringUtil.capitalize("cAt") = "CAt"
+     *
+ * + *

+ * + * @param str + * 要转换的字符串 + * + * @return 首字符为大写的字符串，如果原字符串为null，则返回null + */ + public static String capitalize(String str) { + int strLen; + + if ((str == null) || ((strLen = str.length()) == 0)) { + return str; + } + + return new StringBuffer(strLen).append(Character.toTitleCase(str.charAt(0))).append(str.substring(1)) + .toString(); + } + + + /** + * 将字符串的首字符转成小写，其它字符不变。 + * + *

+ * 如果字符串是null则返回null。 + * + * 

+     * StringUtil.uncapitalize(null)  = null
+     * StringUtil.uncapitalize("")    = ""
+     * StringUtil.uncapitalize("Cat") = "cat"
+     * StringUtil.uncapitalize("CAT") = "cAT"
+     *
+ * + *

+ * + * @param str + * 要转换的字符串 + * + * @return 首字符为小写的字符串，如果原字符串为null，则返回null + */ + public static String uncapitalize(String str) { + int strLen; + + if ((str == null) || ((strLen = str.length()) == 0)) { + return str; + } + + return new StringBuffer(strLen).append(Character.toLowerCase(str.charAt(0))).append(str.substring(1)) + .toString(); + } + + + /** + * 反转字符串的大小写。 + * + *

+ * 如果字符串是null则返回null。 + * + * 

+     * StringUtil.swapCase(null)                 = null
+     * StringUtil.swapCase("")                   = ""
+     * StringUtil.swapCase("The dog has a BONE") = "tHE DOG HAS A bone"
+     *
+ * + *

+ * + * @param str + * 要转换的字符串 + * + * @return 大小写被反转的字符串，如果原字符串为null，则返回null + */ + public static String swapCase(String str) { + int strLen; + + if ((str == null) || ((strLen = str.length()) == 0)) { + return str; + } + + StringBuffer buffer = new StringBuffer(strLen); + + char ch = 0; + + for (int i = 0; i < strLen; i++) { + ch = str.charAt(i); + + if (Character.isUpperCase(ch)) { + ch = Character.toLowerCase(ch); + } + else if (Character.isTitleCase(ch)) { + ch = Character.toLowerCase(ch); + } + else if (Character.isLowerCase(ch)) { + ch = Character.toUpperCase(ch); + } + + buffer.append(ch); + } + + return buffer.toString(); + } + + + /** + * 将字符串转换成camel case。 + * + *

+ * 如果字符串是null则返回null。 + * + * 

+     * StringUtil.toCamelCase(null)  = null
+     * StringUtil.toCamelCase("")    = ""
+     * StringUtil.toCamelCase("aBc") = "aBc"
+     * StringUtil.toCamelCase("aBc def") = "aBcDef"
+     * StringUtil.toCamelCase("aBc def_ghi") = "aBcDefGhi"
+     * StringUtil.toCamelCase("aBc def_ghi 123") = "aBcDefGhi123"
+     *
+ * + *

+ * + *

+ * 此方法会保留除了下划线和空白以外的所有分隔符。 + *

+ * + * @param str + * 要转换的字符串 + * + * @return camel case字符串，如果原字符串为null，则返回null + */ + public static String toCamelCase(String str) { + return CAMEL_CASE_TOKENIZER.parse(str); + } + + + /** + * 将字符串转换成pascal case。 + * + *

+ * 如果字符串是null则返回null。 + * + * 

+     * StringUtil.toPascalCase(null)  = null
+     * StringUtil.toPascalCase("")    = ""
+     * StringUtil.toPascalCase("aBc") = "ABc"
+     * StringUtil.toPascalCase("aBc def") = "ABcDef"
+     * StringUtil.toPascalCase("aBc def_ghi") = "ABcDefGhi"
+     * StringUtil.toPascalCase("aBc def_ghi 123") = "aBcDefGhi123"
+     *
+ * + *

+ * + *

+ * 此方法会保留除了下划线和空白以外的所有分隔符。 + *

+ * + * @param str + * 要转换的字符串 + * + * @return pascal case字符串，如果原字符串为null，则返回null + */ + public static String toPascalCase(String str) { + return PASCAL_CASE_TOKENIZER.parse(str); + } + + + /** + * 将字符串转换成下划线分隔的大写字符串。 + * + *

+ * 如果字符串是null则返回null。 + * + * 

+     * StringUtil.toUpperCaseWithUnderscores(null)  = null
+     * StringUtil.toUpperCaseWithUnderscores("")    = ""
+     * StringUtil.toUpperCaseWithUnderscores("aBc") = "A_BC"
+     * StringUtil.toUpperCaseWithUnderscores("aBc def") = "A_BC_DEF"
+     * StringUtil.toUpperCaseWithUnderscores("aBc def_ghi") = "A_BC_DEF_GHI"
+     * StringUtil.toUpperCaseWithUnderscores("aBc def_ghi 123") = "A_BC_DEF_GHI_123"
+     * StringUtil.toUpperCaseWithUnderscores("__a__Bc__") = "__A__BC__"
+     *
+ * + *

+ * + *

+ * 此方法会保留除了空白以外的所有分隔符。 + *

+ * + * @param str + * 要转换的字符串 + * + * @return 下划线分隔的大写字符串，如果原字符串为null，则返回null + */ + public static String toUpperCaseWithUnderscores(String str) { + return UPPER_CASE_WITH_UNDERSCORES_TOKENIZER.parse(str); + } + + + /** + * 将字符串转换成下划线分隔的小写字符串。 + * + *

+ * 如果字符串是null则返回null。 + * + * 

+     * StringUtil.toLowerCaseWithUnderscores(null)  = null
+     * StringUtil.toLowerCaseWithUnderscores("")    = ""
+     * StringUtil.toLowerCaseWithUnderscores("aBc") = "a_bc"
+     * StringUtil.toLowerCaseWithUnderscores("aBc def") = "a_bc_def"
+     * StringUtil.toLowerCaseWithUnderscores("aBc def_ghi") = "a_bc_def_ghi"
+     * StringUtil.toLowerCaseWithUnderscores("aBc def_ghi 123") = "a_bc_def_ghi_123"
+     * StringUtil.toLowerCaseWithUnderscores("__a__Bc__") = "__a__bc__"
+     *
+ * + *

+ * + *

+ * 此方法会保留除了空白以外的所有分隔符。 + *

+ * + * @param str + * 要转换的字符串 + * + * @return 下划线分隔的小写字符串，如果原字符串为null，则返回null + */ + public static String toLowerCaseWithUnderscores(String str) { + return LOWER_CASE_WITH_UNDERSCORES_TOKENIZER.parse(str); + } + + /** 解析单词的解析器。 */ + private static final WordTokenizer CAMEL_CASE_TOKENIZER = new WordTokenizer() { + protected void startSentence(StringBuffer buffer, char ch) { + buffer.append(Character.toLowerCase(ch)); + } + + + protected void startWord(StringBuffer buffer, char ch) { + if (!isDelimiter(buffer.charAt(buffer.length() - 1))) { + buffer.append(Character.toUpperCase(ch)); + } + else { + buffer.append(Character.toLowerCase(ch)); + } + } + + + protected void inWord(StringBuffer buffer, char ch) { + buffer.append(Character.toLowerCase(ch)); + } + + + protected void startDigitSentence(StringBuffer buffer, char ch) { + buffer.append(ch); + } + + + protected void startDigitWord(StringBuffer buffer, char ch) { + buffer.append(ch); + } + + + protected void inDigitWord(StringBuffer buffer, char ch) { + buffer.append(ch); + } + + + protected void inDelimiter(StringBuffer buffer, char ch) { + if (ch != UNDERSCORE) { + buffer.append(ch); + } + } + }; + + private static final WordTokenizer PASCAL_CASE_TOKENIZER = new WordTokenizer() { + protected void startSentence(StringBuffer buffer, char ch) { + buffer.append(Character.toUpperCase(ch)); + } + + + protected void startWord(StringBuffer buffer, char ch) { + buffer.append(Character.toUpperCase(ch)); + } + + + protected void inWord(StringBuffer buffer, char ch) { + buffer.append(Character.toLowerCase(ch)); + } + + + protected void startDigitSentence(StringBuffer buffer, char ch) { + buffer.append(ch); + } + + + protected void startDigitWord(StringBuffer buffer, char ch) { + buffer.append(ch); + } + + + protected void inDigitWord(StringBuffer buffer, char ch) { + buffer.append(ch); + } + + + protected void inDelimiter(StringBuffer buffer, char ch) { + if (ch != UNDERSCORE) { + buffer.append(ch); + } + } + }; + + private static final WordTokenizer UPPER_CASE_WITH_UNDERSCORES_TOKENIZER = new WordTokenizer() { + protected void startSentence(StringBuffer buffer, char ch) { + buffer.append(Character.toUpperCase(ch)); + } + + + protected void startWord(StringBuffer buffer, char ch) { + if (!isDelimiter(buffer.charAt(buffer.length() - 1))) { + buffer.append(UNDERSCORE); + } + + buffer.append(Character.toUpperCase(ch)); + } + + + protected void inWord(StringBuffer buffer, char ch) { + buffer.append(Character.toUpperCase(ch)); + } + + + protected void startDigitSentence(StringBuffer buffer, char ch) { + buffer.append(ch); + } + + + protected void startDigitWord(StringBuffer buffer, char ch) { + if (!isDelimiter(buffer.charAt(buffer.length() - 1))) { + buffer.append(UNDERSCORE); + } + + buffer.append(ch); + } + + + protected void inDigitWord(StringBuffer buffer, char ch) { + buffer.append(ch); + } + + + protected void inDelimiter(StringBuffer buffer, char ch) { + buffer.append(ch); + } + }; + + private static final WordTokenizer LOWER_CASE_WITH_UNDERSCORES_TOKENIZER = new WordTokenizer() { + protected void startSentence(StringBuffer buffer, char ch) { + buffer.append(Character.toLowerCase(ch)); + } + + + protected void startWord(StringBuffer buffer, char ch) { + if (!isDelimiter(buffer.charAt(buffer.length() - 1))) { + buffer.append(UNDERSCORE); + } + + buffer.append(Character.toLowerCase(ch)); + } + + + protected void inWord(StringBuffer buffer, char ch) { + buffer.append(Character.toLowerCase(ch)); + } + + + protected void startDigitSentence(StringBuffer buffer, char ch) { + buffer.append(ch); + } + + + protected void startDigitWord(StringBuffer buffer, char ch) { + if (!isDelimiter(buffer.charAt(buffer.length() - 1))) { + buffer.append(UNDERSCORE); + } + + buffer.append(ch); + } + + + protected void inDigitWord(StringBuffer buffer, char ch) { + buffer.append(ch); + } + + + protected void inDelimiter(StringBuffer buffer, char ch) { + buffer.append(ch); + } + }; + + /** + * 解析出下列语法所构成的SENTENCE。 + * + * 
+     *  SENTENCE = WORD (DELIMITER* WORD)*
+     * 
+     *  WORD = UPPER_CASE_WORD | LOWER_CASE_WORD | TITLE_CASE_WORD | DIGIT_WORD
+     * 
+     *  UPPER_CASE_WORD = UPPER_CASE_LETTER+
+     *  LOWER_CASE_WORD = LOWER_CASE_LETTER+
+     *  TITLE_CASE_WORD = UPPER_CASE_LETTER LOWER_CASE_LETTER+
+     *  DIGIT_WORD      = DIGIT+
+     * 
+     *  UPPER_CASE_LETTER = Character.isUpperCase()
+     *  LOWER_CASE_LETTER = Character.isLowerCase()
+     *  DIGIT             = Character.isDigit()
+     *  NON_LETTER_DIGIT  = !Character.isUpperCase() && !Character.isLowerCase() && !Character.isDigit()
+     * 
+     *  DELIMITER = WHITESPACE | NON_LETTER_DIGIT
+     *
+ */ + private abstract static class WordTokenizer { + protected static final char UNDERSCORE = '_'; + + + /** + * Parse sentence。 + */ + public String parse(String str) { + if (StringUtil.isEmpty(str)) { + return str; + } + + int length = str.length(); + StringBuffer buffer = new StringBuffer(length); + + for (int index = 0; index < length; index++) { + char ch = str.charAt(index); + + // 忽略空白。 + if (Character.isWhitespace(ch)) { + continue; + } + + // 大写字母开始：UpperCaseWord或是TitleCaseWord。 + if (Character.isUpperCase(ch)) { + int wordIndex = index + 1; + + while (wordIndex < length) { + char wordChar = str.charAt(wordIndex); + + if (Character.isUpperCase(wordChar)) { + wordIndex++; + } + else if (Character.isLowerCase(wordChar)) { + wordIndex--; + break; + } + else { + break; + } + } + + // 1. wordIndex == length，说明最后一个字母为大写，以upperCaseWord处理之。 + // 2. wordIndex == index，说明index处为一个titleCaseWord。 + // 3. wordIndex > index，说明index到wordIndex - + // 1处全部是大写，以upperCaseWord处理。 + if ((wordIndex == length) || (wordIndex > index)) { + index = parseUpperCaseWord(buffer, str, index, wordIndex); + } + else { + index = parseTitleCaseWord(buffer, str, index); + } + + continue; + } + + // 小写字母开始：LowerCaseWord。 + if (Character.isLowerCase(ch)) { + index = parseLowerCaseWord(buffer, str, index); + continue; + } + + // 数字开始：DigitWord。 + if (Character.isDigit(ch)) { + index = parseDigitWord(buffer, str, index); + continue; + } + + // 非字母数字开始：Delimiter。 + inDelimiter(buffer, ch); + } + + return buffer.toString(); + } + + + private int parseUpperCaseWord(StringBuffer buffer, String str, int index, int length) { + char ch = str.charAt(index++); + + // 首字母，必然存在且为大写。 + if (buffer.length() == 0) { + startSentence(buffer, ch); + } + else { + startWord(buffer, ch); + } + + // 后续字母，必为小写。 + for (; index < length; index++) { + ch = str.charAt(index); + inWord(buffer, ch); + } + + return index - 1; + } + + + private int parseLowerCaseWord(StringBuffer buffer, String str, int index) { + char ch = str.charAt(index++); + + // 首字母，必然存在且为小写。 + if (buffer.length() == 0) { + startSentence(buffer, ch); + } + else { + startWord(buffer, ch); + } + + // 后续字母，必为小写。 + int length = str.length(); + + for (; index < length; index++) { + ch = str.charAt(index); + + if (Character.isLowerCase(ch)) { + inWord(buffer, ch); + } + else { + break; + } + } + + return index - 1; + } + + + private int parseTitleCaseWord(StringBuffer buffer, String str, int index) { + char ch = str.charAt(index++); + + // 首字母，必然存在且为大写。 + if (buffer.length() == 0) { + startSentence(buffer, ch); + } + else { + startWord(buffer, ch); + } + + // 后续字母，必为小写。 + int length = str.length(); + + for (; index < length; index++) { + ch = str.charAt(index); + + if (Character.isLowerCase(ch)) { + inWord(buffer, ch); + } + else { + break; + } + } + + return index - 1; + } + + + private int parseDigitWord(StringBuffer buffer, String str, int index) { + char ch = str.charAt(index++); + + // 首字符，必然存在且为数字。 + if (buffer.length() == 0) { + startDigitSentence(buffer, ch); + } + else { + startDigitWord(buffer, ch); + } + + // 后续字符，必为数字。 + int length = str.length(); + + for (; index < length; index++) { + ch = str.charAt(index); + + if (Character.isDigit(ch)) { + inDigitWord(buffer, ch); + } + else { + break; + } + } + + return index - 1; + } + + + protected boolean isDelimiter(char ch) { + return !Character.isUpperCase(ch) && !Character.isLowerCase(ch) && !Character.isDigit(ch); + } + + + protected abstract void startSentence(StringBuffer buffer, char ch); + + + protected abstract void startWord(StringBuffer buffer, char ch); + + + protected abstract void inWord(StringBuffer buffer, char ch); + + + protected abstract void startDigitSentence(StringBuffer buffer, char ch); + + + protected abstract void startDigitWord(StringBuffer buffer, char ch); + + + protected abstract void inDigitWord(StringBuffer buffer, char ch); + + + protected abstract void inDelimiter(StringBuffer buffer, char ch); + } + + + /* + * ========================================================================== + * == + */ + /* 字符串分割函数。 */ + /* */ + /* 将字符串按指定分隔符分割。 */ + /* + * ========================================================================== + * == + */ + + /** + * 将字符串按空白字符分割。 + * + *

+ * 分隔符不会出现在目标数组中，连续的分隔符就被看作一个。如果字符串为null，则返回null。 + * + * 

+     * StringUtil.split(null)       = null
+     * StringUtil.split("")         = []
+     * StringUtil.split("abc def")  = ["abc", "def"]
+     * StringUtil.split("abc  def") = ["abc", "def"]
+     * StringUtil.split(" abc ")    = ["abc"]
+     *
+ * + *

+ * + * @param str + * 要分割的字符串 + * + * @return 分割后的字符串数组，如果原字符串为null，则返回null + */ + public static String[] split(String str) { + return split(str, null, -1); + } + + + /** + * 将字符串按指定字符分割。 + * + *

+ * 分隔符不会出现在目标数组中，连续的分隔符就被看作一个。如果字符串为null，则返回null。 + * + * 

+     * StringUtil.split(null, *)         = null
+     * StringUtil.split("", *)           = []
+     * StringUtil.split("a.b.c", '.')    = ["a", "b", "c"]
+     * StringUtil.split("a..b.c", '.')   = ["a", "b", "c"]
+     * StringUtil.split("a:b:c", '.')    = ["a:b:c"]
+     * StringUtil.split("a b c", ' ')    = ["a", "b", "c"]
+     *
+ * + *

+ * + * @param str + * 要分割的字符串 + * @param separatorChar + * 分隔符 + * + * @return 分割后的字符串数组，如果原字符串为null，则返回null + */ + public static String[] split(String str, char separatorChar) { + if (str == null) { + return null; + } + + int length = str.length(); + + if (length == 0) { + return ArrayUtil.EMPTY_STRING_ARRAY; + } + + List list = new ArrayList(); + int i = 0; + int start = 0; + boolean match = false; + + while (i < length) { + if (str.charAt(i) == separatorChar) { + if (match) { + list.add(str.substring(start, i)); + match = false; + } + + start = ++i; + continue; + } + + match = true; + i++; + } + + if (match) { + list.add(str.substring(start, i)); + } + + return (String[]) list.toArray(new String[list.size()]); + } + + + /** + * 将字符串按指定字符分割。 + * + *

+ * 分隔符不会出现在目标数组中，连续的分隔符就被看作一个。如果字符串为null，则返回null。 + * + * 

+     * StringUtil.split(null, *)                = null
+     * StringUtil.split("", *)                  = []
+     * StringUtil.split("abc def", null)        = ["abc", "def"]
+     * StringUtil.split("abc def", " ")         = ["abc", "def"]
+     * StringUtil.split("abc  def", " ")        = ["abc", "def"]
+     * StringUtil.split(" ab:  cd::ef  ", ":")  = ["ab", "cd", "ef"]
+     * StringUtil.split("abc.def", "")          = ["abc.def"]
+     *
+ * + *

+ * + * @param str + * 要分割的字符串 + * @param separatorChars + * 分隔符 + * + * @return 分割后的字符串数组，如果原字符串为null，则返回null + */ + public static String[] split(String str, String separatorChars) { + return split(str, separatorChars, -1); + } + + + /** + * 将字符串按指定字符分割。 + * + *

+ * 分隔符不会出现在目标数组中，连续的分隔符就被看作一个。如果字符串为null，则返回null。 + * + * 

+     * StringUtil.split(null, *, *)                 = null
+     * StringUtil.split("", *, *)                   = []
+     * StringUtil.split("ab cd ef", null, 0)        = ["ab", "cd", "ef"]
+     * StringUtil.split("  ab   cd ef  ", null, 0)  = ["ab", "cd", "ef"]
+     * StringUtil.split("ab:cd::ef", ":", 0)        = ["ab", "cd", "ef"]
+     * StringUtil.split("ab:cd:ef", ":", 2)         = ["ab", "cdef"]
+     * StringUtil.split("abc.def", "", 2)           = ["abc.def"]
+     *
+ * + *

+ * + * @param str + * 要分割的字符串 + * @param separatorChars + * 分隔符 + * @param max + * 返回的数组的最大个数，如果小于等于0，则表示无限制 + * + * @return 分割后的字符串数组，如果原字符串为null，则返回null + */ + public static String[] split(String str, String separatorChars, int max) { + if (str == null) { + return null; + } + + int length = str.length(); + + if (length == 0) { + return ArrayUtil.EMPTY_STRING_ARRAY; + } + + List list = new ArrayList(); + int sizePlus1 = 1; + int i = 0; + int start = 0; + boolean match = false; + + if (separatorChars == null) { + // null表示使用空白作为分隔符 + while (i < length) { + if (Character.isWhitespace(str.charAt(i))) { + if (match) { + if (sizePlus1++ == max) { + i = length; + } + + list.add(str.substring(start, i)); + match = false; + } + + start = ++i; + continue; + } + + match = true; + i++; + } + } + else if (separatorChars.length() == 1) { + // 优化分隔符长度为1的情形 + char sep = separatorChars.charAt(0); + + while (i < length) { + if (str.charAt(i) == sep) { + if (match) { + if (sizePlus1++ == max) { + i = length; + } + + list.add(str.substring(start, i)); + match = false; + } + + start = ++i; + continue; + } + + match = true; + i++; + } + } + else { + // 一般情形 + while (i < length) { + if (separatorChars.indexOf(str.charAt(i)) >= 0) { + if (match) { + if (sizePlus1++ == max) { + i = length; + } + + list.add(str.substring(start, i)); + match = false; + } + + start = ++i; + continue; + } + + match = true; + i++; + } + } + + if (match) { + list.add(str.substring(start, i)); + } + + return (String[]) list.toArray(new String[list.size()]); + } + + + /* + * ========================================================================== + * == + */ + /* 字符串连接函数。 */ + /* */ + /* 将多个对象按指定分隔符连接成字符串。 */ + /* + * ========================================================================== + * == + */ + + /** + * 将数组中的元素连接成一个字符串。 + * + * 
+     * StringUtil.join(null)            = null
+     * StringUtil.join([])              = ""
+     * StringUtil.join([null])          = ""
+     * StringUtil.join(["a", "b", "c"]) = "abc"
+     * StringUtil.join([null, "", "a"]) = "a"
+     *
+ * + * @param array + * 要连接的数组 + * + * @return 连接后的字符串，如果原数组为null，则返回null + */ + public static String join(Object[] array) { + return join(array, null); + } + + + /** + * 将数组中的元素连接成一个字符串。 + * + * 
+     * StringUtil.join(null, *)               = null
+     * StringUtil.join([], *)                 = ""
+     * StringUtil.join([null], *)             = ""
+     * StringUtil.join(["a", "b", "c"], ';')  = "a;b;c"
+     * StringUtil.join(["a", "b", "c"], null) = "abc"
+     * StringUtil.join([null, "", "a"], ';')  = ";;a"
+     *
+ * + * @param array + * 要连接的数组 + * @param separator + * 分隔符 + * + * @return 连接后的字符串，如果原数组为null，则返回null + */ + public static String join(Object[] array, char separator) { + if (array == null) { + return null; + } + + int arraySize = array.length; + int bufSize = + (arraySize == 0) ? 0 + : ((((array[0] == null) ? 16 : array[0].toString().length()) + 1) * arraySize); + StringBuffer buf = new StringBuffer(bufSize); + + for (int i = 0; i < arraySize; i++) { + if (i > 0) { + buf.append(separator); + } + + if (array[i] != null) { + buf.append(array[i]); + } + } + + return buf.toString(); + } + + + /** + * 将数组中的元素连接成一个字符串。 + * + * 
+     * StringUtil.join(null, *)                = null
+     * StringUtil.join([], *)                  = ""
+     * StringUtil.join([null], *)              = ""
+     * StringUtil.join(["a", "b", "c"], "--")  = "a--b--c"
+     * StringUtil.join(["a", "b", "c"], null)  = "abc"
+     * StringUtil.join(["a", "b", "c"], "")    = "abc"
+     * StringUtil.join([null, "", "a"], ',')   = ",,a"
+     *
+ * + * @param array + * 要连接的数组 + * @param separator + * 分隔符 + * + * @return 连接后的字符串，如果原数组为null，则返回null + */ + public static String join(Object[] array, String separator) { + if (array == null) { + return null; + } + + if (separator == null) { + separator = EMPTY_STRING; + } + + int arraySize = array.length; + + // ArraySize == 0: Len = 0 + // ArraySize > 0: Len = NofStrings *(len(firstString) + len(separator)) + // (估计大约所有的字符串都一样长) + int bufSize = + (arraySize == 0) ? 0 + : (arraySize * (((array[0] == null) ? 16 : array[0].toString().length()) + ((separator != null) ? separator + .length() : 0))); + + StringBuffer buf = new StringBuffer(bufSize); + + for (int i = 0; i < arraySize; i++) { + if ((separator != null) && (i > 0)) { + buf.append(separator); + } + + if (array[i] != null) { + buf.append(array[i]); + } + } + + return buf.toString(); + } + + + /** + * 将Iterator中的元素连接成一个字符串。 + * + * 
+     * StringUtil.join(null, *)                = null
+     * StringUtil.join([], *)                  = ""
+     * StringUtil.join([null], *)              = ""
+     * StringUtil.join(["a", "b", "c"], "--")  = "a--b--c"
+     * StringUtil.join(["a", "b", "c"], null)  = "abc"
+     * StringUtil.join(["a", "b", "c"], "")    = "abc"
+     * StringUtil.join([null, "", "a"], ',')   = ",,a"
+     *
+ * + * @param iterator + * 要连接的Iterator + * @param separator + * 分隔符 + * + * @return 连接后的字符串，如果原数组为null，则返回null + */ + public static String join(Iterator iterator, char separator) { + if (iterator == null) { + return null; + } + + StringBuffer buf = new StringBuffer(256); // Java默认值是16, 可能偏小 + + while (iterator.hasNext()) { + Object obj = iterator.next(); + + if (obj != null) { + buf.append(obj); + } + + if (iterator.hasNext()) { + buf.append(separator); + } + } + + return buf.toString(); + } + + + /** + * 将Iterator中的元素连接成一个字符串。 + * + * 
+     * StringUtil.join(null, *)                = null
+     * StringUtil.join([], *)                  = ""
+     * StringUtil.join([null], *)              = ""
+     * StringUtil.join(["a", "b", "c"], "--")  = "a--b--c"
+     * StringUtil.join(["a", "b", "c"], null)  = "abc"
+     * StringUtil.join(["a", "b", "c"], "")    = "abc"
+     * StringUtil.join([null, "", "a"], ',')   = ",,a"
+     *
+ * + * @param iterator + * 要连接的Iterator + * @param separator + * 分隔符 + * + * @return 连接后的字符串，如果原数组为null，则返回null + */ + public static String join(Iterator iterator, String separator) { + if (iterator == null) { + return null; + } + + StringBuffer buf = new StringBuffer(256); // Java默认值是16, 可能偏小 + + while (iterator.hasNext()) { + Object obj = iterator.next(); + + if (obj != null) { + buf.append(obj); + } + + if ((separator != null) && iterator.hasNext()) { + buf.append(separator); + } + } + + return buf.toString(); + } + + + /* + * ========================================================================== + * == + */ + /* 字符串查找函数 —— 字符或字符串。 */ + /* */ + /* 在字符串中查找指定字符或字符串。 */ + /* + * ========================================================================== + * == + */ + + /** + * 在字符串中查找指定字符，并返回第一个匹配的索引值。如果字符串为null或未找到，则返回-1。 + * + * 
+     * StringUtil.indexOf(null, *)         = -1
+     * StringUtil.indexOf("", *)           = -1
+     * StringUtil.indexOf("aabaabaa", 'a') = 0
+     * StringUtil.indexOf("aabaabaa", 'b') = 2
+     *
+ * + * @param str + * 要扫描的字符串 + * @param searchChar + * 要查找的字符 + * + * @return 第一个匹配的索引值。如果字符串为null或未找到，则返回-1 + */ + public static int indexOf(String str, char searchChar) { + if ((str == null) || (str.length() == 0)) { + return -1; + } + + return str.indexOf(searchChar); + } + + + /** + * 在字符串中查找指定字符，并返回第一个匹配的索引值。如果字符串为null或未找到，则返回-1。 + * + * 
+     * StringUtil.indexOf(null, *, *)          = -1
+     * StringUtil.indexOf("", *, *)            = -1
+     * StringUtil.indexOf("aabaabaa", 'b', 0)  = 2
+     * StringUtil.indexOf("aabaabaa", 'b', 3)  = 5
+     * StringUtil.indexOf("aabaabaa", 'b', 9)  = -1
+     * StringUtil.indexOf("aabaabaa", 'b', -1) = 2
+     *
+ * + * @param str + * 要扫描的字符串 + * @param searchChar + * 要查找的字符 + * @param startPos + * 开始搜索的索引值，如果小于0，则看作0 + * + * @return 第一个匹配的索引值。如果字符串为null或未找到，则返回-1 + */ + public static int indexOf(String str, char searchChar, int startPos) { + if ((str == null) || (str.length() == 0)) { + return -1; + } + + return str.indexOf(searchChar, startPos); + } + + + /** + * 在字符串中查找指定字符串，并返回第一个匹配的索引值。如果字符串为null或未找到，则返回-1。 + * + * 
+     * StringUtil.indexOf(null, *)          = -1
+     * StringUtil.indexOf(*, null)          = -1
+     * StringUtil.indexOf("", "")           = 0
+     * StringUtil.indexOf("aabaabaa", "a")  = 0
+     * StringUtil.indexOf("aabaabaa", "b")  = 2
+     * StringUtil.indexOf("aabaabaa", "ab") = 1
+     * StringUtil.indexOf("aabaabaa", "")   = 0
+     *
+ * + * @param str + * 要扫描的字符串 + * @param searchStr + * 要查找的字符串 + * + * @return 第一个匹配的索引值。如果字符串为null或未找到，则返回-1 + */ + public static int indexOf(String str, String searchStr) { + if ((str == null) || (searchStr == null)) { + return -1; + } + + return str.indexOf(searchStr); + } + + + /** + * 在字符串中查找指定字符串，并返回第一个匹配的索引值。如果字符串为null或未找到，则返回-1。 + * + * 
+     * StringUtil.indexOf(null, *, *)          = -1
+     * StringUtil.indexOf(*, null, *)          = -1
+     * StringUtil.indexOf("", "", 0)           = 0
+     * StringUtil.indexOf("aabaabaa", "a", 0)  = 0
+     * StringUtil.indexOf("aabaabaa", "b", 0)  = 2
+     * StringUtil.indexOf("aabaabaa", "ab", 0) = 1
+     * StringUtil.indexOf("aabaabaa", "b", 3)  = 5
+     * StringUtil.indexOf("aabaabaa", "b", 9)  = -1
+     * StringUtil.indexOf("aabaabaa", "b", -1) = 2
+     * StringUtil.indexOf("aabaabaa", "", 2)   = 2
+     * StringUtil.indexOf("abc", "", 9)        = 3
+     *
+ * + * @param str + * 要扫描的字符串 + * @param searchStr + * 要查找的字符串 + * @param startPos + * 开始搜索的索引值，如果小于0，则看作0 + * + * @return 第一个匹配的索引值。如果字符串为null或未找到，则返回-1 + */ + public static int indexOf(String str, String searchStr, int startPos) { + if ((str == null) || (searchStr == null)) { + return -1; + } + + // JDK1.3及以下版本的bug：不能正确处理下面的情况 + if ((searchStr.length() == 0) && (startPos >= str.length())) { + return str.length(); + } + + return str.indexOf(searchStr, startPos); + } + + + /** + * 在字符串中查找指定字符集合中的字符，并返回第一个匹配的起始索引。 如果字符串为null，则返回 + * -1。 如果字符集合为null或空，也返回-1。 + * + * 
+     * StringUtil.indexOfAny(null, *)                = -1
+     * StringUtil.indexOfAny("", *)                  = -1
+     * StringUtil.indexOfAny(*, null)                = -1
+     * StringUtil.indexOfAny(*, [])                  = -1
+     * StringUtil.indexOfAny("zzabyycdxx",['z','a']) = 0
+     * StringUtil.indexOfAny("zzabyycdxx",['b','y']) = 3
+     * StringUtil.indexOfAny("aba", ['z'])           = -1
+     *
+ * + * @param str + * 要扫描的字符串 + * @param searchChars + * 要搜索的字符集合 + * + * @return 第一个匹配的索引值。如果字符串为null或未找到，则返回-1 + */ + public static int indexOfAny(String str, char[] searchChars) { + if ((str == null) || (str.length() == 0) || (searchChars == null) || (searchChars.length == 0)) { + return -1; + } + + for (int i = 0; i < str.length(); i++) { + char ch = str.charAt(i); + + for (int j = 0; j < searchChars.length; j++) { + if (searchChars[j] == ch) { + return i; + } + } + } + + return -1; + } + + + /** + * 在字符串中查找指定字符集合中的字符，并返回第一个匹配的起始索引。 如果字符串为null，则返回 + * -1。 如果字符集合为null或空，也返回-1。 + * + * 
+     * StringUtil.indexOfAny(null, *)            = -1
+     * StringUtil.indexOfAny("", *)              = -1
+     * StringUtil.indexOfAny(*, null)            = -1
+     * StringUtil.indexOfAny(*, "")              = -1
+     * StringUtil.indexOfAny("zzabyycdxx", "za") = 0
+     * StringUtil.indexOfAny("zzabyycdxx", "by") = 3
+     * StringUtil.indexOfAny("aba","z")          = -1
+     *
+ * + * @param str + * 要扫描的字符串 + * @param searchChars + * 要搜索的字符集合 + * + * @return 第一个匹配的索引值。如果字符串为null或未找到，则返回-1 + */ + public static int indexOfAny(String str, String searchChars) { + if ((str == null) || (str.length() == 0) || (searchChars == null) || (searchChars.length() == 0)) { + return -1; + } + + for (int i = 0; i < str.length(); i++) { + char ch = str.charAt(i); + + for (int j = 0; j < searchChars.length(); j++) { + if (searchChars.charAt(j) == ch) { + return i; + } + } + } + + return -1; + } + + + /** + * 在字符串中查找指定字符串集合中的字符串，并返回第一个匹配的起始索引。 如果字符串为null，则返回 + * -1。 如果字符串集合为null或空，也返回-1。 + * 如果字符串集合包括""，并且字符串不为null，则返回 + * str.length() + * + * 
+     * StringUtil.indexOfAny(null, *)                     = -1
+     * StringUtil.indexOfAny(*, null)                     = -1
+     * StringUtil.indexOfAny(*, [])                       = -1
+     * StringUtil.indexOfAny("zzabyycdxx", ["ab","cd"])   = 2
+     * StringUtil.indexOfAny("zzabyycdxx", ["cd","ab"])   = 2
+     * StringUtil.indexOfAny("zzabyycdxx", ["mn","op"])   = -1
+     * StringUtil.indexOfAny("zzabyycdxx", ["zab","aby"]) = 1
+     * StringUtil.indexOfAny("zzabyycdxx", [""])          = 0
+     * StringUtil.indexOfAny("", [""])                    = 0
+     * StringUtil.indexOfAny("", ["a"])                   = -1
+     *
+ * + * @param str + * 要扫描的字符串 + * @param searchStrs + * 要搜索的字符串集合 + * + * @return 第一个匹配的索引值。如果字符串为null或未找到，则返回-1 + */ + public static int indexOfAny(String str, String[] searchStrs) { + if ((str == null) || (searchStrs == null)) { + return -1; + } + + int sz = searchStrs.length; + + // String's can't have a MAX_VALUEth index. + int ret = Integer.MAX_VALUE; + + int tmp = 0; + + for (int i = 0; i < sz; i++) { + String search = searchStrs[i]; + + if (search == null) { + continue; + } + + tmp = str.indexOf(search); + + if (tmp == -1) { + continue; + } + + if (tmp < ret) { + ret = tmp; + } + } + + return (ret == Integer.MAX_VALUE) ? (-1) : ret; + } + + + /** + * 在字符串中查找不在指定字符集合中的字符，并返回第一个匹配的起始索引。 如果字符串为null，则返回 + * -1。 如果字符集合为null或空，也返回-1。 + * + * 
+     * StringUtil.indexOfAnyBut(null, *)             = -1
+     * StringUtil.indexOfAnyBut("", *)               = -1
+     * StringUtil.indexOfAnyBut(*, null)             = -1
+     * StringUtil.indexOfAnyBut(*, [])               = -1
+     * StringUtil.indexOfAnyBut("zzabyycdxx",'za')   = 3
+     * StringUtil.indexOfAnyBut("zzabyycdxx", 'by')  = 0
+     * StringUtil.indexOfAnyBut("aba", 'ab')         = -1
+     *
+ * + * @param str + * 要扫描的字符串 + * @param searchChars + * 要搜索的字符集合 + * + * @return 第一个匹配的索引值。如果字符串为null或未找到，则返回-1 + */ + public static int indexOfAnyBut(String str, char[] searchChars) { + if ((str == null) || (str.length() == 0) || (searchChars == null) || (searchChars.length == 0)) { + return -1; + } + + outer: for (int i = 0; i < str.length(); i++) { + char ch = str.charAt(i); + + for (int j = 0; j < searchChars.length; j++) { + if (searchChars[j] == ch) { + continue outer; + } + } + + return i; + } + + return -1; + } + + + /** + * 在字符串中查找不在指定字符集合中的字符，并返回第一个匹配的起始索引。 如果字符串为null，则返回 + * -1。 如果字符集合为null或空，也返回-1。 + * + * 
+     * StringUtil.indexOfAnyBut(null, *)            = -1
+     * StringUtil.indexOfAnyBut("", *)              = -1
+     * StringUtil.indexOfAnyBut(*, null)            = -1
+     * StringUtil.indexOfAnyBut(*, "")              = -1
+     * StringUtil.indexOfAnyBut("zzabyycdxx", "za") = 3
+     * StringUtil.indexOfAnyBut("zzabyycdxx", "by") = 0
+     * StringUtil.indexOfAnyBut("aba","ab")         = -1
+     *
+ * + * @param str + * 要扫描的字符串 + * @param searchChars + * 要搜索的字符集合 + * + * @return 第一个匹配的索引值。如果字符串为null或未找到，则返回-1 + */ + public static int indexOfAnyBut(String str, String searchChars) { + if ((str == null) || (str.length() == 0) || (searchChars == null) || (searchChars.length() == 0)) { + return -1; + } + + for (int i = 0; i < str.length(); i++) { + if (searchChars.indexOf(str.charAt(i)) < 0) { + return i; + } + } + + return -1; + } + + + /** + * 从字符串尾部开始查找指定字符，并返回第一个匹配的索引值。如果字符串为null或未找到，则返回 + * -1。 + * + * 
+     * StringUtil.lastIndexOf(null, *)         = -1
+     * StringUtil.lastIndexOf("", *)           = -1
+     * StringUtil.lastIndexOf("aabaabaa", 'a') = 7
+     * StringUtil.lastIndexOf("aabaabaa", 'b') = 5
+     *
+ * + * @param str + * 要扫描的字符串 + * @param searchChar + * 要查找的字符 + * + * @return 第一个匹配的索引值。如果字符串为null或未找到，则返回-1 + */ + public static int lastIndexOf(String str, char searchChar) { + if ((str == null) || (str.length() == 0)) { + return -1; + } + + return str.lastIndexOf(searchChar); + } + + + /** + * 从字符串尾部开始查找指定字符，并返回第一个匹配的索引值。如果字符串为null或未找到，则返回 + * -1。 + * + * 
+     * StringUtil.lastIndexOf(null, *, *)          = -1
+     * StringUtil.lastIndexOf("", *,  *)           = -1
+     * StringUtil.lastIndexOf("aabaabaa", 'b', 8)  = 5
+     * StringUtil.lastIndexOf("aabaabaa", 'b', 4)  = 2
+     * StringUtil.lastIndexOf("aabaabaa", 'b', 0)  = -1
+     * StringUtil.lastIndexOf("aabaabaa", 'b', 9)  = 5
+     * StringUtil.lastIndexOf("aabaabaa", 'b', -1) = -1
+     * StringUtil.lastIndexOf("aabaabaa", 'a', 0)  = 0
+     *
+ * + * @param str + * 要扫描的字符串 + * @param searchChar + * 要查找的字符 + * @param startPos + * 从指定索引开始向前搜索 + * + * @return 第一个匹配的索引值。如果字符串为null或未找到，则返回-1 + */ + public static int lastIndexOf(String str, char searchChar, int startPos) { + if ((str == null) || (str.length() == 0)) { + return -1; + } + + return str.lastIndexOf(searchChar, startPos); + } + + + /** + * 从字符串尾部开始查找指定字符串，并返回第一个匹配的索引值。如果字符串为null或未找到，则返回 + * -1。 + * + * 
+     * StringUtil.lastIndexOf(null, *)         = -1
+     * StringUtil.lastIndexOf("", *)           = -1
+     * StringUtil.lastIndexOf("aabaabaa", 'a') = 7
+     * StringUtil.lastIndexOf("aabaabaa", 'b') = 5
+     *
+ * + * @param str + * 要扫描的字符串 + * @param searchStr + * 要查找的字符串 + * + * @return 第一个匹配的索引值。如果字符串为null或未找到，则返回-1 + */ + public static int lastIndexOf(String str, String searchStr) { + if ((str == null) || (searchStr == null)) { + return -1; + } + + return str.lastIndexOf(searchStr); + } + + + /** + * 从字符串尾部开始查找指定字符串，并返回第一个匹配的索引值。如果字符串为null或未找到，则返回 + * -1。 + * + * 
+     * StringUtil.lastIndexOf(null, *, *)          = -1
+     * StringUtil.lastIndexOf(*, null, *)          = -1
+     * StringUtil.lastIndexOf("aabaabaa", "a", 8)  = 7
+     * StringUtil.lastIndexOf("aabaabaa", "b", 8)  = 5
+     * StringUtil.lastIndexOf("aabaabaa", "ab", 8) = 4
+     * StringUtil.lastIndexOf("aabaabaa", "b", 9)  = 5
+     * StringUtil.lastIndexOf("aabaabaa", "b", -1) = -1
+     * StringUtil.lastIndexOf("aabaabaa", "a", 0)  = 0
+     * StringUtil.lastIndexOf("aabaabaa", "b", 0)  = -1
+     *
+ * + * @param str + * 要扫描的字符串 + * @param searchStr + * 要查找的字符串 + * @param startPos + * 从指定索引开始向前搜索 + * + * @return 第一个匹配的索引值。如果字符串为null或未找到，则返回-1 + */ + public static int lastIndexOf(String str, String searchStr, int startPos) { + if ((str == null) || (searchStr == null)) { + return -1; + } + + return str.lastIndexOf(searchStr, startPos); + } + + + /** + * 从字符串尾部开始查找指定字符串集合中的字符串，并返回第一个匹配的起始索引。 如果字符串为null，则返回 + * -1。 如果字符串集合为null或空，也返回-1。 + * 如果字符串集合包括""，并且字符串不为null，则返回 + * str.length() + * + * 
+     * StringUtil.lastIndexOfAny(null, *)                   = -1
+     * StringUtil.lastIndexOfAny(*, null)                   = -1
+     * StringUtil.lastIndexOfAny(*, [])                     = -1
+     * StringUtil.lastIndexOfAny(*, [null])                 = -1
+     * StringUtil.lastIndexOfAny("zzabyycdxx", ["ab","cd"]) = 6
+     * StringUtil.lastIndexOfAny("zzabyycdxx", ["cd","ab"]) = 6
+     * StringUtil.lastIndexOfAny("zzabyycdxx", ["mn","op"]) = -1
+     * StringUtil.lastIndexOfAny("zzabyycdxx", ["mn","op"]) = -1
+     * StringUtil.lastIndexOfAny("zzabyycdxx", ["mn",""])   = 10
+     *
+ * + * @param str + * 要扫描的字符串 + * @param searchStrs + * 要搜索的字符串集合 + * + * @return 第一个匹配的索引值。如果字符串为null或未找到，则返回-1 + */ + public static int lastIndexOfAny(String str, String[] searchStrs) { + if ((str == null) || (searchStrs == null)) { + return -1; + } + + int searchStrsLength = searchStrs.length; + int index = -1; + int tmp = 0; + + for (int i = 0; i < searchStrsLength; i++) { + String search = searchStrs[i]; + + if (search == null) { + continue; + } + + tmp = str.lastIndexOf(search); + + if (tmp > index) { + index = tmp; + } + } + + return index; + } + + + /** + * 检查字符串中是否包含指定的字符。如果字符串为null，将返回false。 + * + * 
+     * StringUtil.contains(null, *)    = false
+     * StringUtil.contains("", *)      = false
+     * StringUtil.contains("abc", 'a') = true
+     * StringUtil.contains("abc", 'z') = false
+     *
+ * + * @param str + * 要扫描的字符串 + * @param searchChar + * 要查找的字符 + * + * @return 如果找到，则返回true + */ + public static boolean contains(String str, char searchChar) { + if ((str == null) || (str.length() == 0)) { + return false; + } + + return str.indexOf(searchChar) >= 0; + } + + + /** + * 检查字符串中是否包含指定的字符串。如果字符串为null，将返回false。 + * + * 
+     * StringUtil.contains(null, *)     = false
+     * StringUtil.contains(*, null)     = false
+     * StringUtil.contains("", "")      = true
+     * StringUtil.contains("abc", "")   = true
+     * StringUtil.contains("abc", "a")  = true
+     * StringUtil.contains("abc", "z")  = false
+     *
+ * + * @param str + * 要扫描的字符串 + * @param searchStr + * 要查找的字符串 + * + * @return 如果找到，则返回true + */ + public static boolean contains(String str, String searchStr) { + if ((str == null) || (searchStr == null)) { + return false; + } + + return str.indexOf(searchStr) >= 0; + } + + + /** + * 检查字符串是是否只包含指定字符集合中的字符。 + * + *

+ * 如果字符串为null，则返回false。 如果字符集合为null + * 则返回false。 但是空字符串永远返回true. + *

+ * + * 
+     * StringUtil.containsOnly(null, *)       = false
+     * StringUtil.containsOnly(*, null)       = false
+     * StringUtil.containsOnly("", *)         = true
+     * StringUtil.containsOnly("ab", '')      = false
+     * StringUtil.containsOnly("abab", 'abc') = true
+     * StringUtil.containsOnly("ab1", 'abc')  = false
+     * StringUtil.containsOnly("abz", 'abc')  = false
+     *
+ * + * @param str + * 要扫描的字符串 + * @param valid + * 要查找的字符串 + * + * @return 如果找到，则返回true + */ + public static boolean containsOnly(String str, char[] valid) { + if ((valid == null) || (str == null)) { + return false; + } + + if (str.length() == 0) { + return true; + } + + if (valid.length == 0) { + return false; + } + + return indexOfAnyBut(str, valid) == -1; + } + + + /** + * 检查字符串是是否只包含指定字符集合中的字符。 + * + *

+ * 如果字符串为null，则返回false。 如果字符集合为null + * 则返回false。 但是空字符串永远返回true. + *

+ * + * 
+     * StringUtil.containsOnly(null, *)       = false
+     * StringUtil.containsOnly(*, null)       = false
+     * StringUtil.containsOnly("", *)         = true
+     * StringUtil.containsOnly("ab", "")      = false
+     * StringUtil.containsOnly("abab", "abc") = true
+     * StringUtil.containsOnly("ab1", "abc")  = false
+     * StringUtil.containsOnly("abz", "abc")  = false
+     *
+ * + * @param str + * 要扫描的字符串 + * @param valid + * 要查找的字符串 + * + * @return 如果找到，则返回true + */ + public static boolean containsOnly(String str, String valid) { + if ((str == null) || (valid == null)) { + return false; + } + + return containsOnly(str, valid.toCharArray()); + } + + + /** + * 检查字符串是是否不包含指定字符集合中的字符。 + * + *

+ * 如果字符串为null，则返回false。 如果字符集合为null + * 则返回true。 但是空字符串永远返回true. + *

+ * + * 
+     * StringUtil.containsNone(null, *)       = true
+     * StringUtil.containsNone(*, null)       = true
+     * StringUtil.containsNone("", *)         = true
+     * StringUtil.containsNone("ab", '')      = true
+     * StringUtil.containsNone("abab", 'xyz') = true
+     * StringUtil.containsNone("ab1", 'xyz')  = true
+     * StringUtil.containsNone("abz", 'xyz')  = false
+     *
+ * + * @param str + * 要扫描的字符串 + * @param invalid + * 要查找的字符串 + * + * @return 如果找到，则返回true + */ + public static boolean containsNone(String str, char[] invalid) { + if ((str == null) || (invalid == null)) { + return true; + } + + int strSize = str.length(); + int validSize = invalid.length; + + for (int i = 0; i < strSize; i++) { + char ch = str.charAt(i); + + for (int j = 0; j < validSize; j++) { + if (invalid[j] == ch) { + return false; + } + } + } + + return true; + } + + + /** + * 检查字符串是是否不包含指定字符集合中的字符。 + * + *

+ * 如果字符串为null，则返回false。 如果字符集合为null + * 则返回true。 但是空字符串永远返回true. + *

+ * + * 
+     * StringUtil.containsNone(null, *)       = true
+     * StringUtil.containsNone(*, null)       = true
+     * StringUtil.containsNone("", *)         = true
+     * StringUtil.containsNone("ab", "")      = true
+     * StringUtil.containsNone("abab", "xyz") = true
+     * StringUtil.containsNone("ab1", "xyz")  = true
+     * StringUtil.containsNone("abz", "xyz")  = false
+     *
+ * + * @param str + * 要扫描的字符串 + * @param invalidChars + * 要查找的字符串 + * + * @return 如果找到，则返回true + */ + public static boolean containsNone(String str, String invalidChars) { + if ((str == null) || (invalidChars == null)) { + return true; + } + + return containsNone(str, invalidChars.toCharArray()); + } + + + /** + * 取得指定子串在字符串中出现的次数。 + * + *

+ * 如果字符串为null或空，则返回0。 + * + * 

+     * StringUtil.countMatches(null, *)       = 0
+     * StringUtil.countMatches("", *)         = 0
+     * StringUtil.countMatches("abba", null)  = 0
+     * StringUtil.countMatches("abba", "")    = 0
+     * StringUtil.countMatches("abba", "a")   = 2
+     * StringUtil.countMatches("abba", "ab")  = 1
+     * StringUtil.countMatches("abba", "xxx") = 0
+     *
+ * + *

+ * + * @param str + * 要扫描的字符串 + * @param subStr + * 子字符串 + * + * @return 子串在字符串中出现的次数，如果字符串为null或空，则返回0 + */ + public static int countMatches(String str, String subStr) { + if ((str == null) || (str.length() == 0) || (subStr == null) || (subStr.length() == 0)) { + return 0; + } + + int count = 0; + int index = 0; + + while ((index = str.indexOf(subStr, index)) != -1) { + count++; + index += subStr.length(); + } + + return count; + } + + + /* + * ========================================================================== + * == + */ + /* 取子串函数。 */ + /* + * ========================================================================== + * == + */ + + /** + * 取指定字符串的子串。 + * + *

+ * 负的索引代表从尾部开始计算。如果字符串为null，则返回null。 + * + * 

+     * StringUtil.substring(null, *)   = null
+     * StringUtil.substring("", *)     = ""
+     * StringUtil.substring("abc", 0)  = "abc"
+     * StringUtil.substring("abc", 2)  = "c"
+     * StringUtil.substring("abc", 4)  = ""
+     * StringUtil.substring("abc", -2) = "bc"
+     * StringUtil.substring("abc", -4) = "abc"
+     *
+ * + *

+ * + * @param str + * 字符串 + * @param start + * 起始索引，如果为负数，表示从尾部查找 + * + * @return 子串，如果原始串为null，则返回null + */ + public static String substring(String str, int start) { + if (str == null) { + return null; + } + + if (start < 0) { + start = str.length() + start; + } + + if (start < 0) { + start = 0; + } + + if (start > str.length()) { + return EMPTY_STRING; + } + + return str.substring(start); + } + + + /** + * 取指定字符串的子串。 + * + *

+ * 负的索引代表从尾部开始计算。如果字符串为null，则返回null。 + * + * 

+     * StringUtil.substring(null, *, *)    = null
+     * StringUtil.substring("", * ,  *)    = "";
+     * StringUtil.substring("abc", 0, 2)   = "ab"
+     * StringUtil.substring("abc", 2, 0)   = ""
+     * StringUtil.substring("abc", 2, 4)   = "c"
+     * StringUtil.substring("abc", 4, 6)   = ""
+     * StringUtil.substring("abc", 2, 2)   = ""
+     * StringUtil.substring("abc", -2, -1) = "b"
+     * StringUtil.substring("abc", -4, 2)  = "ab"
+     *
+ * + *

+ * + * @param str + * 字符串 + * @param start + * 起始索引，如果为负数，表示从尾部计算 + * @param end + * 结束索引（不含），如果为负数，表示从尾部计算 + * + * @return 子串，如果原始串为null，则返回null + */ + public static String substring(String str, int start, int end) { + if (str == null) { + return null; + } + + if (end < 0) { + end = str.length() + end; + } + + if (start < 0) { + start = str.length() + start; + } + + if (end > str.length()) { + end = str.length(); + } + + if (start > end) { + return EMPTY_STRING; + } + + if (start < 0) { + start = 0; + } + + if (end < 0) { + end = 0; + } + + return str.substring(start, end); + } + + + /** + * 取得长度为指定字符数的最左边的子串。 + * + * 
+     * StringUtil.left(null, *)    = null
+     * StringUtil.left(*, -ve)     = ""
+     * StringUtil.left("", *)      = ""
+     * StringUtil.left("abc", 0)   = ""
+     * StringUtil.left("abc", 2)   = "ab"
+     * StringUtil.left("abc", 4)   = "abc"
+     *
+ * + * @param str + * 字符串 + * @param len + * 最左子串的长度 + * + * @return 子串，如果原始字串为null，则返回null + */ + public static String left(String str, int len) { + if (str == null) { + return null; + } + + if (len < 0) { + return EMPTY_STRING; + } + + if (str.length() <= len) { + return str; + } + else { + return str.substring(0, len); + } + } + + + /** + * 取得长度为指定字符数的最右边的子串。 + * + * 
+     * StringUtil.right(null, *)    = null
+     * StringUtil.right(*, -ve)     = ""
+     * StringUtil.right("", *)      = ""
+     * StringUtil.right("abc", 0)   = ""
+     * StringUtil.right("abc", 2)   = "bc"
+     * StringUtil.right("abc", 4)   = "abc"
+     *
+ * + * @param str + * 字符串 + * @param len + * 最右子串的长度 + * + * @return 子串，如果原始字串为null，则返回null + */ + public static String right(String str, int len) { + if (str == null) { + return null; + } + + if (len < 0) { + return EMPTY_STRING; + } + + if (str.length() <= len) { + return str; + } + else { + return str.substring(str.length() - len); + } + } + + + /** + * 取得从指定索引开始计算的、长度为指定字符数的子串。 + * + * 
+     * StringUtil.mid(null, *, *)    = null
+     * StringUtil.mid(*, *, -ve)     = ""
+     * StringUtil.mid("", 0, *)      = ""
+     * StringUtil.mid("abc", 0, 2)   = "ab"
+     * StringUtil.mid("abc", 0, 4)   = "abc"
+     * StringUtil.mid("abc", 2, 4)   = "c"
+     * StringUtil.mid("abc", 4, 2)   = ""
+     * StringUtil.mid("abc", -2, 2)  = "ab"
+     *
+ * + * @param str + * 字符串 + * @param pos + * 起始索引，如果为负数，则看作0 + * @param len + * 子串的长度，如果为负数，则看作长度为0 + * + * @return 子串，如果原始字串为null，则返回null + */ + public static String mid(String str, int pos, int len) { + if (str == null) { + return null; + } + + if ((len < 0) || (pos > str.length())) { + return EMPTY_STRING; + } + + if (pos < 0) { + pos = 0; + } + + if (str.length() <= (pos + len)) { + return str.substring(pos); + } + else { + return str.substring(pos, pos + len); + } + } + + + /* + * ========================================================================== + * == + */ + /* 搜索并取子串函数。 */ + /* + * ========================================================================== + * == + */ + + /** + * 取得第一个出现的分隔子串之前的子串。 + * + *

+ * 如果字符串为null，则返回null。 如果分隔子串为null + * 或未找到该子串，则返回原字符串。 + * + * 

+     * StringUtil.substringBefore(null, *)      = null
+     * StringUtil.substringBefore("", *)        = ""
+     * StringUtil.substringBefore("abc", "a")   = ""
+     * StringUtil.substringBefore("abcba", "b") = "a"
+     * StringUtil.substringBefore("abc", "c")   = "ab"
+     * StringUtil.substringBefore("abc", "d")   = "abc"
+     * StringUtil.substringBefore("abc", "")    = ""
+     * StringUtil.substringBefore("abc", null)  = "abc"
+     *
+ * + *

+ * + * @param str + * 字符串 + * @param separator + * 要搜索的分隔子串 + * + * @return 子串，如果原始串为null，则返回null + */ + public static String substringBefore(String str, String separator) { + if ((str == null) || (separator == null) || (str.length() == 0)) { + return str; + } + + if (separator.length() == 0) { + return EMPTY_STRING; + } + + int pos = str.indexOf(separator); + + if (pos == -1) { + return str; + } + + return str.substring(0, pos); + } + + + /** + * 取得第一个出现的分隔子串之后的子串。 + * + *

+ * 如果字符串为null，则返回null。 如果分隔子串为null + * 或未找到该子串，则返回原字符串。 + * + * 

+     * StringUtil.substringAfter(null, *)      = null
+     * StringUtil.substringAfter("", *)        = ""
+     * StringUtil.substringAfter(*, null)      = ""
+     * StringUtil.substringAfter("abc", "a")   = "bc"
+     * StringUtil.substringAfter("abcba", "b") = "cba"
+     * StringUtil.substringAfter("abc", "c")   = ""
+     * StringUtil.substringAfter("abc", "d")   = ""
+     * StringUtil.substringAfter("abc", "")    = "abc"
+     *
+ * + *

+ * + * @param str + * 字符串 + * @param separator + * 要搜索的分隔子串 + * + * @return 子串，如果原始串为null，则返回null + */ + public static String substringAfter(String str, String separator) { + if ((str == null) || (str.length() == 0)) { + return str; + } + + if (separator == null) { + return EMPTY_STRING; + } + + int pos = str.indexOf(separator); + + if (pos == -1) { + return EMPTY_STRING; + } + + return str.substring(pos + separator.length()); + } + + + /** + * 取得最后一个的分隔子串之前的子串。 + * + *

+ * 如果字符串为null，则返回null。 如果分隔子串为null + * 或未找到该子串，则返回原字符串。 + * + * 

+     * StringUtil.substringBeforeLast(null, *)      = null
+     * StringUtil.substringBeforeLast("", *)        = ""
+     * StringUtil.substringBeforeLast("abcba", "b") = "abc"
+     * StringUtil.substringBeforeLast("abc", "c")   = "ab"
+     * StringUtil.substringBeforeLast("a", "a")     = ""
+     * StringUtil.substringBeforeLast("a", "z")     = "a"
+     * StringUtil.substringBeforeLast("a", null)    = "a"
+     * StringUtil.substringBeforeLast("a", "")      = "a"
+     *
+ * + *

+ * + * @param str + * 字符串 + * @param separator + * 要搜索的分隔子串 + * + * @return 子串，如果原始串为null，则返回null + */ + public static String substringBeforeLast(String str, String separator) { + if ((str == null) || (separator == null) || (str.length() == 0) || (separator.length() == 0)) { + return str; + } + + int pos = str.lastIndexOf(separator); + + if (pos == -1) { + return str; + } + + return str.substring(0, pos); + } + + + /** + * 取得最后一个的分隔子串之后的子串。 + * + *

+ * 如果字符串为null，则返回null。 如果分隔子串为null + * 或未找到该子串，则返回原字符串。 + * + * 

+     * StringUtil.substringAfterLast(null, *)      = null
+     * StringUtil.substringAfterLast("", *)        = ""
+     * StringUtil.substringAfterLast(*, "")        = ""
+     * StringUtil.substringAfterLast(*, null)      = ""
+     * StringUtil.substringAfterLast("abc", "a")   = "bc"
+     * StringUtil.substringAfterLast("abcba", "b") = "a"
+     * StringUtil.substringAfterLast("abc", "c")   = ""
+     * StringUtil.substringAfterLast("a", "a")     = ""
+     * StringUtil.substringAfterLast("a", "z")     = ""
+     *
+ * + *

+ * + * @param str + * 字符串 + * @param separator + * 要搜索的分隔子串 + * + * @return 子串，如果原始串为null，则返回null + */ + public static String substringAfterLast(String str, String separator) { + if ((str == null) || (str.length() == 0)) { + return str; + } + + if ((separator == null) || (separator.length() == 0)) { + return EMPTY_STRING; + } + + int pos = str.lastIndexOf(separator); + + if ((pos == -1) || (pos == (str.length() - separator.length()))) { + return EMPTY_STRING; + } + + return str.substring(pos + separator.length()); + } + + + /** + * 取得指定分隔符的前两次出现之间的子串。 + * + *

+ * 如果字符串为null，则返回null。 如果分隔子串为null + * ，则返回null。 + * + * 

+     * StringUtil.substringBetween(null, *)            = null
+     * StringUtil.substringBetween("", "")             = ""
+     * StringUtil.substringBetween("", "tag")          = null
+     * StringUtil.substringBetween("tagabctag", null)  = null
+     * StringUtil.substringBetween("tagabctag", "")    = ""
+     * StringUtil.substringBetween("tagabctag", "tag") = "abc"
+     *
+ * + *

+ * + * @param str + * 字符串 + * @param tag + * 要搜索的分隔子串 + * + * @return 子串，如果原始串为null或未找到分隔子串，则返回null + */ + public static String substringBetween(String str, String tag) { + return substringBetween(str, tag, tag, 0); + } + + + /** + * 取得两个分隔符之间的子串。 + * + *

+ * 如果字符串为null，则返回null。 如果分隔子串为null + * ，则返回null。 + * + * 

+     * StringUtil.substringBetween(null, *, *)          = null
+     * StringUtil.substringBetween("", "", "")          = ""
+     * StringUtil.substringBetween("", "", "tag")       = null
+     * StringUtil.substringBetween("", "tag", "tag")    = null
+     * StringUtil.substringBetween("yabcz", null, null) = null
+     * StringUtil.substringBetween("yabcz", "", "")     = ""
+     * StringUtil.substringBetween("yabcz", "y", "z")   = "abc"
+     * StringUtil.substringBetween("yabczyabcz", "y", "z")   = "abc"
+     *
+ * + *

+ * + * @param str + * 字符串 + * @param open + * 要搜索的分隔子串1 + * @param close + * 要搜索的分隔子串2 + * + * @return 子串，如果原始串为null或未找到分隔子串，则返回null + */ + public static String substringBetween(String str, String open, String close) { + return substringBetween(str, open, close, 0); + } + + + /** + * 取得两个分隔符之间的子串。 + * + *

+ * 如果字符串为null，则返回null。 如果分隔子串为null + * ，则返回null。 + * + * 

+     * StringUtil.substringBetween(null, *, *)          = null
+     * StringUtil.substringBetween("", "", "")          = ""
+     * StringUtil.substringBetween("", "", "tag")       = null
+     * StringUtil.substringBetween("", "tag", "tag")    = null
+     * StringUtil.substringBetween("yabcz", null, null) = null
+     * StringUtil.substringBetween("yabcz", "", "")     = ""
+     * StringUtil.substringBetween("yabcz", "y", "z")   = "abc"
+     * StringUtil.substringBetween("yabczyabcz", "y", "z")   = "abc"
+     *
+ * + *

+ * + * @param str + * 字符串 + * @param open + * 要搜索的分隔子串1 + * @param close + * 要搜索的分隔子串2 + * @param fromIndex + * 从指定index处搜索 + * + * @return 子串，如果原始串为null或未找到分隔子串，则返回null + */ + public static String substringBetween(String str, String open, String close, int fromIndex) { + if ((str == null) || (open == null) || (close == null)) { + return null; + } + + int start = str.indexOf(open, fromIndex); + + if (start != -1) { + int end = str.indexOf(close, start + open.length()); + + if (end != -1) { + return str.substring(start + open.length(), end); + } + } + + return null; + } + + + /* + * ========================================================================== + * == + */ + /* 删除字符。 */ + /* + * ========================================================================== + * == + */ + + /** + * 删除所有在Character.isWhitespace(char)中所定义的空白。 + * + * 
+     * StringUtil.deleteWhitespace(null)         = null
+     * StringUtil.deleteWhitespace("")           = ""
+     * StringUtil.deleteWhitespace("abc")        = "abc"
+     * StringUtil.deleteWhitespace("   ab  c  ") = "abc"
+     *
+ * + * @param str + * 要处理的字符串 + * + * @return 去空白后的字符串，如果原始字符串为null，则返回null + */ + public static String deleteWhitespace(String str) { + if (str == null) { + return null; + } + + int sz = str.length(); + StringBuffer buffer = new StringBuffer(sz); + + for (int i = 0; i < sz; i++) { + if (!Character.isWhitespace(str.charAt(i))) { + buffer.append(str.charAt(i)); + } + } + + return buffer.toString(); + } + + + /* + * ========================================================================== + * == + */ + /* 替换子串。 */ + /* + * ========================================================================== + * == + */ + + /** + * 替换指定的子串，只替换第一个出现的子串。 + * + *

+ * 如果字符串为null则返回null，如果指定子串为null + * ，则返回原字符串。 + * + * 

+     * StringUtil.replaceOnce(null, *, *)        = null
+     * StringUtil.replaceOnce("", *, *)          = ""
+     * StringUtil.replaceOnce("aba", null, null) = "aba"
+     * StringUtil.replaceOnce("aba", null, null) = "aba"
+     * StringUtil.replaceOnce("aba", "a", null)  = "aba"
+     * StringUtil.replaceOnce("aba", "a", "")    = "ba"
+     * StringUtil.replaceOnce("aba", "a", "z")   = "zba"
+     *
+ * + *

+ * + * @param text + * 要扫描的字符串 + * @param repl + * 要搜索的子串 + * @param with + * 替换字符串 + * + * @return 被替换后的字符串，如果原始字符串为null，则返回null + */ + public static String replaceOnce(String text, String repl, String with) { + return replace(text, repl, with, 1); + } + + + /** + * 替换指定的子串，替换所有出现的子串。 + * + *

+ * 如果字符串为null则返回null，如果指定子串为null + * ，则返回原字符串。 + * + * 

+     * StringUtil.replace(null, *, *)        = null
+     * StringUtil.replace("", *, *)          = ""
+     * StringUtil.replace("aba", null, null) = "aba"
+     * StringUtil.replace("aba", null, null) = "aba"
+     * StringUtil.replace("aba", "a", null)  = "aba"
+     * StringUtil.replace("aba", "a", "")    = "b"
+     * StringUtil.replace("aba", "a", "z")   = "zbz"
+     *
+ * + *

+ * + * @param text + * 要扫描的字符串 + * @param repl + * 要搜索的子串 + * @param with + * 替换字符串 + * + * @return 被替换后的字符串，如果原始字符串为null，则返回null + */ + public static String replace(String text, String repl, String with) { + return replace(text, repl, with, -1); + } + + + /** + * 替换指定的子串，替换指定的次数。 + * + *

+ * 如果字符串为null则返回null，如果指定子串为null + * ，则返回原字符串。 + * + * 

+     * StringUtil.replace(null, *, *, *)         = null
+     * StringUtil.replace("", *, *, *)           = ""
+     * StringUtil.replace("abaa", null, null, 1) = "abaa"
+     * StringUtil.replace("abaa", null, null, 1) = "abaa"
+     * StringUtil.replace("abaa", "a", null, 1)  = "abaa"
+     * StringUtil.replace("abaa", "a", "", 1)    = "baa"
+     * StringUtil.replace("abaa", "a", "z", 0)   = "abaa"
+     * StringUtil.replace("abaa", "a", "z", 1)   = "zbaa"
+     * StringUtil.replace("abaa", "a", "z", 2)   = "zbza"
+     * StringUtil.replace("abaa", "a", "z", -1)  = "zbzz"
+     *
+ * + *

+ * + * @param text + * 要扫描的字符串 + * @param repl + * 要搜索的子串 + * @param with + * 替换字符串 + * @param max + * maximum number of values to replace, or -1 if no + * maximum + * + * @return 被替换后的字符串，如果原始字符串为null，则返回null + */ + public static String replace(String text, String repl, String with, int max) { + if ((text == null) || (repl == null) || (with == null) || (repl.length() == 0) || (max == 0)) { + return text; + } + + StringBuffer buf = new StringBuffer(text.length()); + int start = 0; + int end = 0; + + while ((end = text.indexOf(repl, start)) != -1) { + buf.append(text.substring(start, end)).append(with); + start = end + repl.length(); + + if (--max == 0) { + break; + } + } + + buf.append(text.substring(start)); + return buf.toString(); + } + + + /** + * 将字符串中所有指定的字符，替换成另一个。 + * + *

+ * 如果字符串为null则返回null。 + * + * 

+     * StringUtil.replaceChars(null, *, *)        = null
+     * StringUtil.replaceChars("", *, *)          = ""
+     * StringUtil.replaceChars("abcba", 'b', 'y') = "aycya"
+     * StringUtil.replaceChars("abcba", 'z', 'y') = "abcba"
+     *
+ * + *

+ * + * @param str + * 要扫描的字符串 + * @param searchChar + * 要搜索的字符 + * @param replaceChar + * 替换字符 + * + * @return 被替换后的字符串，如果原始字符串为null，则返回null + */ + public static String replaceChars(String str, char searchChar, char replaceChar) { + if (str == null) { + return null; + } + + return str.replace(searchChar, replaceChar); + } + + + /** + * 将字符串中所有指定的字符，替换成另一个。 + * + *

+ * 如果字符串为null则返回null。如果搜索字符串为null + * 或空，则返回原字符串。 + *

+ * + *

+ * 例如： + * replaceChars("hello", "ho", "jy") = jelly + * 。 + *

+ * + *

+ * 通常搜索字符串和替换字符串是等长的，如果搜索字符串比替换字符串长，则多余的字符将被删除。 如果搜索字符串比替换字符串短，则缺少的字符将被忽略。 + * + * 

+     * StringUtil.replaceChars(null, *, *)           = null
+     * StringUtil.replaceChars("", *, *)             = ""
+     * StringUtil.replaceChars("abc", null, *)       = "abc"
+     * StringUtil.replaceChars("abc", "", *)         = "abc"
+     * StringUtil.replaceChars("abc", "b", null)     = "ac"
+     * StringUtil.replaceChars("abc", "b", "")       = "ac"
+     * StringUtil.replaceChars("abcba", "bc", "yz")  = "ayzya"
+     * StringUtil.replaceChars("abcba", "bc", "y")   = "ayya"
+     * StringUtil.replaceChars("abcba", "bc", "yzx") = "ayzya"
+     *
+ * + *

+ * + * @param str + * 要扫描的字符串 + * @param searchChars + * 要搜索的字符串 + * @param replaceChars + * 替换字符串 + * + * @return 被替换后的字符串，如果原始字符串为null，则返回null + */ + public static String replaceChars(String str, String searchChars, String replaceChars) { + if ((str == null) || (str.length() == 0) || (searchChars == null) || (searchChars.length() == 0)) { + return str; + } + + char[] chars = str.toCharArray(); + int len = chars.length; + boolean modified = false; + + for (int i = 0, isize = searchChars.length(); i < isize; i++) { + char searchChar = searchChars.charAt(i); + + if ((replaceChars == null) || (i >= replaceChars.length())) { + // 删除 + int pos = 0; + + for (int j = 0; j < len; j++) { + if (chars[j] != searchChar) { + chars[pos++] = chars[j]; + } + else { + modified = true; + } + } + + len = pos; + } + else { + // 替换 + for (int j = 0; j < len; j++) { + if (chars[j] == searchChar) { + chars[j] = replaceChars.charAt(i); + modified = true; + } + } + } + } + + if (!modified) { + return str; + } + + return new String(chars, 0, len); + } + + + /** + * 将指定的子串用另一指定子串覆盖。 + * + *

+ * 如果字符串为null，则返回null。 负的索引值将被看作0 + * ，越界的索引值将被设置成字符串的长度相同的值。 + * + * 

+     * StringUtil.overlay(null, *, *, *)            = null
+     * StringUtil.overlay("", "abc", 0, 0)          = "abc"
+     * StringUtil.overlay("abcdef", null, 2, 4)     = "abef"
+     * StringUtil.overlay("abcdef", "", 2, 4)       = "abef"
+     * StringUtil.overlay("abcdef", "", 4, 2)       = "abef"
+     * StringUtil.overlay("abcdef", "zzzz", 2, 4)   = "abzzzzef"
+     * StringUtil.overlay("abcdef", "zzzz", 4, 2)   = "abzzzzef"
+     * StringUtil.overlay("abcdef", "zzzz", -1, 4)  = "zzzzef"
+     * StringUtil.overlay("abcdef", "zzzz", 2, 8)   = "abzzzz"
+     * StringUtil.overlay("abcdef", "zzzz", -2, -3) = "zzzzabcdef"
+     * StringUtil.overlay("abcdef", "zzzz", 8, 10)  = "abcdefzzzz"
+     *
+ * + *

+ * + * @param str + * 要扫描的字符串 + * @param overlay + * 用来覆盖的字符串 + * @param start + * 起始索引 + * @param end + * 结束索引 + * + * @return 被覆盖后的字符串，如果原始字符串为null，则返回null + */ + public static String overlay(String str, String overlay, int start, int end) { + if (str == null) { + return null; + } + + if (overlay == null) { + overlay = EMPTY_STRING; + } + + int len = str.length(); + + if (start < 0) { + start = 0; + } + + if (start > len) { + start = len; + } + + if (end < 0) { + end = 0; + } + + if (end > len) { + end = len; + } + + if (start > end) { + int temp = start; + + start = end; + end = temp; + } + + return new StringBuffer((len + start) - end + overlay.length() + 1).append(str.substring(0, start)) + .append(overlay).append(str.substring(end)).toString(); + } + + + /* + * ========================================================================== + * == + */ + /* Perl风格的chomp和chop函数。 */ + /* + * ========================================================================== + * == + */ + + /** + * 删除字符串末尾的换行符。如果字符串不以换行结尾，则什么也不做。 + * + *

+ * 换行符有三种情形："\n"、"\r"、" + * \r\n"。 + * + * 

+     * StringUtil.chomp(null)          = null
+     * StringUtil.chomp("")            = ""
+     * StringUtil.chomp("abc \r")      = "abc "
+     * StringUtil.chomp("abc\n")       = "abc"
+     * StringUtil.chomp("abc\r\n")     = "abc"
+     * StringUtil.chomp("abc\r\n\r\n") = "abc\r\n"
+     * StringUtil.chomp("abc\n\r")     = "abc\n"
+     * StringUtil.chomp("abc\n\rabc")  = "abc\n\rabc"
+     * StringUtil.chomp("\r")          = ""
+     * StringUtil.chomp("\n")          = ""
+     * StringUtil.chomp("\r\n")        = ""
+     *
+ * + *

+ * + * @param str + * 要处理的字符串 + * + * @return 不以换行结尾的字符串，如果原始字串为null，则返回null + */ + public static String chomp(String str) { + if ((str == null) || (str.length() == 0)) { + return str; + } + + if (str.length() == 1) { + char ch = str.charAt(0); + + if ((ch == '\r') || (ch == '\n')) { + return EMPTY_STRING; + } + else { + return str; + } + } + + int lastIdx = str.length() - 1; + char last = str.charAt(lastIdx); + + if (last == '\n') { + if (str.charAt(lastIdx - 1) == '\r') { + lastIdx--; + } + } + else if (last == '\r') { + } + else { + lastIdx++; + } + + return str.substring(0, lastIdx); + } + + + /** + * 删除字符串末尾的指定字符串。如果字符串不以该字符串结尾，则什么也不做。 + * + * 
+     * StringUtil.chomp(null, *)         = null
+     * StringUtil.chomp("", *)           = ""
+     * StringUtil.chomp("foobar", "bar") = "foo"
+     * StringUtil.chomp("foobar", "baz") = "foobar"
+     * StringUtil.chomp("foo", "foo")    = ""
+     * StringUtil.chomp("foo ", "foo")   = "foo "
+     * StringUtil.chomp(" foo", "foo")   = " "
+     * StringUtil.chomp("foo", "foooo")  = "foo"
+     * StringUtil.chomp("foo", "")       = "foo"
+     * StringUtil.chomp("foo", null)     = "foo"
+     *
+ * + * @param str + * 要处理的字符串 + * @param separator + * 要删除的字符串 + * + * @return 不以指定字符串结尾的字符串，如果原始字串为null，则返回null + */ + public static String chomp(String str, String separator) { + if ((str == null) || (str.length() == 0) || (separator == null)) { + return str; + } + + if (str.endsWith(separator)) { + return str.substring(0, str.length() - separator.length()); + } + + return str; + } + + + /** + * 删除最后一个字符。 + * + *

+ * 如果字符串以\r\n结尾，则同时删除它们。 + * + * 

+     * StringUtil.chop(null)          = null
+     * StringUtil.chop("")            = ""
+     * StringUtil.chop("abc \r")      = "abc "
+     * StringUtil.chop("abc\n")       = "abc"
+     * StringUtil.chop("abc\r\n")     = "abc"
+     * StringUtil.chop("abc")         = "ab"
+     * StringUtil.chop("abc\nabc")    = "abc\nab"
+     * StringUtil.chop("a")           = ""
+     * StringUtil.chop("\r")          = ""
+     * StringUtil.chop("\n")          = ""
+     * StringUtil.chop("\r\n")        = ""
+     *
+ * + *

+ * + * @param str + * 要处理的字符串 + * + * @return 删除最后一个字符的字符串，如果原始字符串为null，则返回null + */ + public static String chop(String str) { + if (str == null) { + return null; + } + + int strLen = str.length(); + + if (strLen < 2) { + return EMPTY_STRING; + } + + int lastIdx = strLen - 1; + String ret = str.substring(0, lastIdx); + char last = str.charAt(lastIdx); + + if (last == '\n') { + if (ret.charAt(lastIdx - 1) == '\r') { + return ret.substring(0, lastIdx - 1); + } + } + + return ret; + } + + + /* + * ========================================================================== + * == + */ + /* 重复/对齐字符串。 */ + /* + * ========================================================================== + * == + */ + + /** + * 将指定字符串重复n遍。 + * + * 
+     * StringUtil.repeat(null, 2)   = null
+     * StringUtil.repeat("", 0)     = ""
+     * StringUtil.repeat("", 2)     = ""
+     * StringUtil.repeat("a", 3)    = "aaa"
+     * StringUtil.repeat("ab", 2)   = "abab"
+     * StringUtil.repeat("abcd", 2) = "abcdabcd"
+     * StringUtil.repeat("a", -2)   = ""
+     *
+ * + * @param str + * 要重复的字符串 + * @param repeat + * 重复次数，如果小于0，则看作0 + * + * @return 重复n次的字符串，如果原始字符串为null，则返回null + */ + public static String repeat(String str, int repeat) { + if (str == null) { + return null; + } + + if (repeat <= 0) { + return EMPTY_STRING; + } + + int inputLength = str.length(); + + if ((repeat == 1) || (inputLength == 0)) { + return str; + } + + int outputLength = inputLength * repeat; + + switch (inputLength) { + case 1: + + char ch = str.charAt(0); + char[] output1 = new char[outputLength]; + + for (int i = repeat - 1; i >= 0; i--) { + output1[i] = ch; + } + + return new String(output1); + + case 2: + + char ch0 = str.charAt(0); + char ch1 = str.charAt(1); + char[] output2 = new char[outputLength]; + + for (int i = (repeat * 2) - 2; i >= 0; i--, i--) { + output2[i] = ch0; + output2[i + 1] = ch1; + } + + return new String(output2); + + default: + + StringBuffer buf = new StringBuffer(outputLength); + + for (int i = 0; i < repeat; i++) { + buf.append(str); + } + + return buf.toString(); + } + } + + + /** + * 扩展并左对齐字符串，用空格' '填充右边。 + * + * 
+     * StringUtil.alignLeft(null, *)   = null
+     * StringUtil.alignLeft("", 3)     = "   "
+     * StringUtil.alignLeft("bat", 3)  = "bat"
+     * StringUtil.alignLeft("bat", 5)  = "bat  "
+     * StringUtil.alignLeft("bat", 1)  = "bat"
+     * StringUtil.alignLeft("bat", -1) = "bat"
+     *
+ * + * @param str + * 要对齐的字符串 + * @param size + * 扩展字符串到指定宽度 + * + * @return 扩展后的字符串，如果字符串为null，则返回null + */ + public static String alignLeft(String str, int size) { + return alignLeft(str, size, ' '); + } + + + /** + * 扩展并左对齐字符串，用指定字符填充右边。 + * + * 
+     * StringUtil.alignLeft(null, *, *)     = null
+     * StringUtil.alignLeft("", 3, 'z')     = "zzz"
+     * StringUtil.alignLeft("bat", 3, 'z')  = "bat"
+     * StringUtil.alignLeft("bat", 5, 'z')  = "batzz"
+     * StringUtil.alignLeft("bat", 1, 'z')  = "bat"
+     * StringUtil.alignLeft("bat", -1, 'z') = "bat"
+     *
+ * + * @param str + * 要对齐的字符串 + * @param size + * 扩展字符串到指定宽度 + * @param padChar + * 填充字符 + * + * @return 扩展后的字符串，如果字符串为null，则返回null + */ + public static String alignLeft(String str, int size, char padChar) { + if (str == null) { + return null; + } + + int pads = size - str.length(); + + if (pads <= 0) { + return str; + } + + return alignLeft(str, size, String.valueOf(padChar)); + } + + + /** + * 扩展并左对齐字符串，用指定字符串填充右边。 + * + * 
+     * StringUtil.alignLeft(null, *, *)      = null
+     * StringUtil.alignLeft("", 3, "z")      = "zzz"
+     * StringUtil.alignLeft("bat", 3, "yz")  = "bat"
+     * StringUtil.alignLeft("bat", 5, "yz")  = "batyz"
+     * StringUtil.alignLeft("bat", 8, "yz")  = "batyzyzy"
+     * StringUtil.alignLeft("bat", 1, "yz")  = "bat"
+     * StringUtil.alignLeft("bat", -1, "yz") = "bat"
+     * StringUtil.alignLeft("bat", 5, null)  = "bat  "
+     * StringUtil.alignLeft("bat", 5, "")    = "bat  "
+     *
+ * + * @param str + * 要对齐的字符串 + * @param size + * 扩展字符串到指定宽度 + * @param padStr + * 填充字符串 + * + * @return 扩展后的字符串，如果字符串为null，则返回null + */ + public static String alignLeft(String str, int size, String padStr) { + if (str == null) { + return null; + } + + if ((padStr == null) || (padStr.length() == 0)) { + padStr = " "; + } + + int padLen = padStr.length(); + int strLen = str.length(); + int pads = size - strLen; + + if (pads <= 0) { + return str; + } + + if (pads == padLen) { + return str.concat(padStr); + } + else if (pads < padLen) { + return str.concat(padStr.substring(0, pads)); + } + else { + char[] padding = new char[pads]; + char[] padChars = padStr.toCharArray(); + + for (int i = 0; i < pads; i++) { + padding[i] = padChars[i % padLen]; + } + + return str.concat(new String(padding)); + } + } + + + /** + * 扩展并右对齐字符串，用空格' '填充左边。 + * + * 
+     * StringUtil.alignRight(null, *)   = null
+     * StringUtil.alignRight("", 3)     = "   "
+     * StringUtil.alignRight("bat", 3)  = "bat"
+     * StringUtil.alignRight("bat", 5)  = "  bat"
+     * StringUtil.alignRight("bat", 1)  = "bat"
+     * StringUtil.alignRight("bat", -1) = "bat"
+     *
+ * + * @param str + * 要对齐的字符串 + * @param size + * 扩展字符串到指定宽度 + * + * @return 扩展后的字符串，如果字符串为null，则返回null + */ + public static String alignRight(String str, int size) { + return alignRight(str, size, ' '); + } + + + /** + * 扩展并右对齐字符串，用指定字符填充左边。 + * + * 
+     * StringUtil.alignRight(null, *, *)     = null
+     * StringUtil.alignRight("", 3, 'z')     = "zzz"
+     * StringUtil.alignRight("bat", 3, 'z')  = "bat"
+     * StringUtil.alignRight("bat", 5, 'z')  = "zzbat"
+     * StringUtil.alignRight("bat", 1, 'z')  = "bat"
+     * StringUtil.alignRight("bat", -1, 'z') = "bat"
+     *
+ * + * @param str + * 要对齐的字符串 + * @param size + * 扩展字符串到指定宽度 + * @param padChar + * 填充字符 + * + * @return 扩展后的字符串，如果字符串为null，则返回null + */ + public static String alignRight(String str, int size, char padChar) { + if (str == null) { + return null; + } + + int pads = size - str.length(); + + if (pads <= 0) { + return str; + } + + return alignRight(str, size, String.valueOf(padChar)); + } + + + /** + * 扩展并右对齐字符串，用指定字符串填充左边。 + * + * 
+     * StringUtil.alignRight(null, *, *)      = null
+     * StringUtil.alignRight("", 3, "z")      = "zzz"
+     * StringUtil.alignRight("bat", 3, "yz")  = "bat"
+     * StringUtil.alignRight("bat", 5, "yz")  = "yzbat"
+     * StringUtil.alignRight("bat", 8, "yz")  = "yzyzybat"
+     * StringUtil.alignRight("bat", 1, "yz")  = "bat"
+     * StringUtil.alignRight("bat", -1, "yz") = "bat"
+     * StringUtil.alignRight("bat", 5, null)  = "  bat"
+     * StringUtil.alignRight("bat", 5, "")    = "  bat"
+     *
+ * + * @param str + * 要对齐的字符串 + * @param size + * 扩展字符串到指定宽度 + * @param padStr + * 填充字符串 + * + * @return 扩展后的字符串，如果字符串为null，则返回null + */ + public static String alignRight(String str, int size, String padStr) { + if (str == null) { + return null; + } + + if ((padStr == null) || (padStr.length() == 0)) { + padStr = " "; + } + + int padLen = padStr.length(); + int strLen = str.length(); + int pads = size - strLen; + + if (pads <= 0) { + return str; + } + + if (pads == padLen) { + return padStr.concat(str); + } + else if (pads < padLen) { + return padStr.substring(0, pads).concat(str); + } + else { + char[] padding = new char[pads]; + char[] padChars = padStr.toCharArray(); + + for (int i = 0; i < pads; i++) { + padding[i] = padChars[i % padLen]; + } + + return new String(padding).concat(str); + } + } + + + /** + * 扩展并居中字符串，用空格' '填充两边。 + * + * 
+     * StringUtil.center(null, *)   = null
+     * StringUtil.center("", 4)     = "    "
+     * StringUtil.center("ab", -1)  = "ab"
+     * StringUtil.center("ab", 4)   = " ab "
+     * StringUtil.center("abcd", 2) = "abcd"
+     * StringUtil.center("a", 4)    = " a  "
+     *
+ * + * @param str + * 要对齐的字符串 + * @param size + * 扩展字符串到指定宽度 + * + * @return 扩展后的字符串，如果字符串为null，则返回null + */ + public static String center(String str, int size) { + return center(str, size, ' '); + } + + + /** + * 扩展并居中字符串，用指定字符填充两边。 + * + * 
+     * StringUtil.center(null, *, *)     = null
+     * StringUtil.center("", 4, ' ')     = "    "
+     * StringUtil.center("ab", -1, ' ')  = "ab"
+     * StringUtil.center("ab", 4, ' ')   = " ab "
+     * StringUtil.center("abcd", 2, ' ') = "abcd"
+     * StringUtil.center("a", 4, ' ')    = " a  "
+     * StringUtil.center("a", 4, 'y')    = "yayy"
+     *
+ * + * @param str + * 要对齐的字符串 + * @param size + * 扩展字符串到指定宽度 + * @param padChar + * 填充字符 + * + * @return 扩展后的字符串，如果字符串为null，则返回null + */ + public static String center(String str, int size, char padChar) { + if ((str == null) || (size <= 0)) { + return str; + } + + int strLen = str.length(); + int pads = size - strLen; + + if (pads <= 0) { + return str; + } + + str = alignRight(str, strLen + (pads / 2), padChar); + str = alignLeft(str, size, padChar); + return str; + } + + + /** + * 扩展并居中字符串，用指定字符串填充两边。 + * + * 
+     * StringUtil.center(null, *, *)     = null
+     * StringUtil.center("", 4, " ")     = "    "
+     * StringUtil.center("ab", -1, " ")  = "ab"
+     * StringUtil.center("ab", 4, " ")   = " ab "
+     * StringUtil.center("abcd", 2, " ") = "abcd"
+     * StringUtil.center("a", 4, " ")    = " a  "
+     * StringUtil.center("a", 4, "yz")   = "yayz"
+     * StringUtil.center("abc", 7, null) = "  abc  "
+     * StringUtil.center("abc", 7, "")   = "  abc  "
+     *
+ * + * @param str + * 要对齐的字符串 + * @param size + * 扩展字符串到指定宽度 + * @param padStr + * 填充字符串 + * + * @return 扩展后的字符串，如果字符串为null，则返回null + */ + public static String center(String str, int size, String padStr) { + if ((str == null) || (size <= 0)) { + return str; + } + + if ((padStr == null) || (padStr.length() == 0)) { + padStr = " "; + } + + int strLen = str.length(); + int pads = size - strLen; + + if (pads <= 0) { + return str; + } + + str = alignRight(str, strLen + (pads / 2), padStr); + str = alignLeft(str, size, padStr); + return str; + } + + + /* + * ========================================================================== + * == + */ + /* 反转字符串。 */ + /* + * ========================================================================== + * == + */ + + /** + * 反转字符串中的字符顺序。 + * + *

+ * 如果字符串为null，则返回null。 + *

+ * + * 
+     * StringUtil.reverse(null)  = null
+     * StringUtil.reverse("")    = ""
+     * StringUtil.reverse("bat") = "tab"
+     *
+ * + * @param str + * 要反转的字符串 + * + * @return 反转后的字符串，如果原字符串为null，则返回null + */ + public static String reverse(String str) { + if ((str == null) || (str.length() == 0)) { + return str; + } + + return new StringBuffer(str).reverse().toString(); + } + + + /** + * 反转指定分隔符分隔的各子串的顺序。 + * + *

+ * 如果字符串为null，则返回null。 + *

+ * + * 
+     * StringUtil.reverseDelimited(null, *)      = null
+     * StringUtil.reverseDelimited("", *)        = ""
+     * StringUtil.reverseDelimited("a.b.c", 'x') = "a.b.c"
+     * StringUtil.reverseDelimited("a.b.c", '.') = "c.b.a"
+     *
+ * + * @param str + * 要反转的字符串 + * @param separatorChar + * 分隔符 + * + * @return 反转后的字符串，如果原字符串为null，则返回null + */ + public static String reverseDelimited(String str, char separatorChar) { + if (str == null) { + return null; + } + + String[] strs = split(str, separatorChar); + + ArrayUtil.reverse(strs); + + return join(strs, separatorChar); + } + + + /** + * 反转指定分隔符分隔的各子串的顺序。 + * + *

+ * 如果字符串为null，则返回null。 + *

+ * + * 
+     * StringUtil.reverseDelimited(null, *, *)          = null
+     * StringUtil.reverseDelimited("", *, *)            = ""
+     * StringUtil.reverseDelimited("a.b.c", null, null) = "a.b.c"
+     * StringUtil.reverseDelimited("a.b.c", "", null)   = "a.b.c"
+     * StringUtil.reverseDelimited("a.b.c", ".", ",")   = "c,b,a"
+     * StringUtil.reverseDelimited("a.b.c", ".", null)  = "c b a"
+     *
+ * + * @param str + * 要反转的字符串 + * @param separatorChars + * 分隔符，如果为null，则默认使用空白字符 + * @param separator + * 用来连接子串的分隔符，如果为null，默认使用空格 + * + * @return 反转后的字符串，如果原字符串为null，则返回null + */ + public static String reverseDelimited(String str, String separatorChars, String separator) { + if (str == null) { + return null; + } + + String[] strs = split(str, separatorChars); + + ArrayUtil.reverse(strs); + + if (separator == null) { + return join(strs, ' '); + } + + return join(strs, separator); + } + + + /* + * ========================================================================== + * == + */ + /* 取得字符串的缩略。 */ + /* + * ========================================================================== + * == + */ + + /** + * 将字符串转换成指定长度的缩略，例如： + * 将"Now is the time for all good men"转换成"Now is the time for..."。 + * + *
    + *
  • + * 如果str比maxWidth短，直接返回；
    • + *
  • + * 否则将它转换成缩略：substring(str, 0, max-3) + "..."；
    • + *
  • + * 如果maxWidth小于4抛出 + * IllegalArgumentException；
    • + *
  • + * 返回的字符串不可能长于指定的maxWidth。
    • + *
+ * + * 
+     * StringUtil.abbreviate(null, *)      = null
+     * StringUtil.abbreviate("", 4)        = ""
+     * StringUtil.abbreviate("abcdefg", 6) = "abc..."
+     * StringUtil.abbreviate("abcdefg", 7) = "abcdefg"
+     * StringUtil.abbreviate("abcdefg", 8) = "abcdefg"
+     * StringUtil.abbreviate("abcdefg", 4) = "a..."
+     * StringUtil.abbreviate("abcdefg", 3) = IllegalArgumentException
+     *
+ * + * @param str + * 要检查的字符串 + * @param maxWidth + * 最大长度，不小于4，如果小于4，则看作4 + * + * @return 字符串缩略，如果原始字符串为null则返回null + */ + public static String abbreviate(String str, int maxWidth) { + return abbreviate(str, 0, maxWidth); + } + + + /** + * 将字符串转换成指定长度的缩略，例如： + * 将"Now is the time for all good men"转换成"...is the time for..."。 + * + *

+ * 和abbreviate(String, int)类似，但是增加了一个“左边界”偏移量。 + * 注意，“左边界”处的字符未必出现在结果字符串的最左边，但一定出现在结果字符串中。 + *

+ * + *

+ * 返回的字符串不可能长于指定的maxWidth。 + * + * 

+     * StringUtil.abbreviate(null, *, *)                = null
+     * StringUtil.abbreviate("", 0, 4)                  = ""
+     * StringUtil.abbreviate("abcdefghijklmno", -1, 10) = "abcdefg..."
+     * StringUtil.abbreviate("abcdefghijklmno", 0, 10)  = "abcdefg..."
+     * StringUtil.abbreviate("abcdefghijklmno", 1, 10)  = "abcdefg..."
+     * StringUtil.abbreviate("abcdefghijklmno", 4, 10)  = "abcdefg..."
+     * StringUtil.abbreviate("abcdefghijklmno", 5, 10)  = "...fghi..."
+     * StringUtil.abbreviate("abcdefghijklmno", 6, 10)  = "...ghij..."
+     * StringUtil.abbreviate("abcdefghijklmno", 8, 10)  = "...ijklmno"
+     * StringUtil.abbreviate("abcdefghijklmno", 10, 10) = "...ijklmno"
+     * StringUtil.abbreviate("abcdefghijklmno", 12, 10) = "...ijklmno"
+     * StringUtil.abbreviate("abcdefghij", 0, 3)        = IllegalArgumentException
+     * StringUtil.abbreviate("abcdefghij", 5, 6)        = IllegalArgumentException
+     *
+ * + *

+ * + * @param str + * 要检查的字符串 + * @param offset + * 左边界偏移量 + * @param maxWidth + * 最大长度，不小于4，如果小于4，则看作4 + * + * @return 字符串缩略，如果原始字符串为null则返回null + */ + public static String abbreviate(String str, int offset, int maxWidth) { + if (str == null) { + return null; + } + + // 调整最大宽度 + if (maxWidth < 4) { + maxWidth = 4; + } + + if (str.length() <= maxWidth) { + return str; + } + + if (offset > str.length()) { + offset = str.length(); + } + + if ((str.length() - offset) < (maxWidth - 3)) { + offset = str.length() - (maxWidth - 3); + } + + if (offset <= 4) { + return str.substring(0, maxWidth - 3) + "..."; + } + + // 调整最大宽度 + if (maxWidth < 7) { + maxWidth = 7; + } + + if ((offset + (maxWidth - 3)) < str.length()) { + return "..." + abbreviate(str.substring(offset), maxWidth - 3); + } + + return "..." + str.substring(str.length() - (maxWidth - 3)); + } + + + /* + * ========================================================================== + * == + */ + /* 比较两个字符串的异同。 */ + /* */ + /* 查找字符串之间的差异，比较字符串的相似度。 */ + /* + * ========================================================================== + * == + */ + + /** + * 比较两个字符串，取得第二个字符串中，和第一个字符串不同的部分。 + * + * 
+     * StringUtil.difference("i am a machine", "i am a robot")  = "robot"
+     * StringUtil.difference(null, null)                        = null
+     * StringUtil.difference("", "")                            = ""
+     * StringUtil.difference("", null)                          = ""
+     * StringUtil.difference("", "abc")                         = "abc"
+     * StringUtil.difference("abc", "")                         = ""
+     * StringUtil.difference("abc", "abc")                      = ""
+     * StringUtil.difference("ab", "abxyz")                     = "xyz"
+     * StringUtil.difference("abcde", "abxyz")                  = "xyz"
+     * StringUtil.difference("abcde", "xyz")                    = "xyz"
+     *
+ * + * @param str1 + * 字符串1 + * @param str2 + * 字符串2 + * + * @return 第二个字符串中，和第一个字符串不同的部分。如果两个字符串相同，则返回空字符串"" + */ + public static String difference(String str1, String str2) { + if (str1 == null) { + return str2; + } + + if (str2 == null) { + return str1; + } + + int index = indexOfDifference(str1, str2); + + if (index == -1) { + return EMPTY_STRING; + } + + return str2.substring(index); + } + + + /** + * 比较两个字符串，取得两字符串开始不同的索引值。 + * + * 
+     * StringUtil.indexOfDifference("i am a machine", "i am a robot")   = 7
+     * StringUtil.indexOfDifference(null, null)                         = -1
+     * StringUtil.indexOfDifference("", null)                           = -1
+     * StringUtil.indexOfDifference("", "")                             = -1
+     * StringUtil.indexOfDifference("", "abc")                          = 0
+     * StringUtil.indexOfDifference("abc", "")                          = 0
+     * StringUtil.indexOfDifference("abc", "abc")                       = -1
+     * StringUtil.indexOfDifference("ab", "abxyz")                      = 2
+     * StringUtil.indexOfDifference("abcde", "abxyz")                   = 2
+     * StringUtil.indexOfDifference("abcde", "xyz")                     = 0
+     *
+ * + * @param str1 + * 字符串1 + * @param str2 + * 字符串2 + * + * @return 两字符串开始产生差异的索引值，如果两字符串相同，则返回-1 + */ + public static int indexOfDifference(String str1, String str2) { + if ((str1 == str2) || (str1 == null) || (str2 == null)) { + return -1; + } + + int i; + + for (i = 0; (i < str1.length()) && (i < str2.length()); ++i) { + if (str1.charAt(i) != str2.charAt(i)) { + break; + } + } + + if ((i < str2.length()) || (i < str1.length())) { + return i; + } + + return -1; + } + + + /** + * 取得两个字符串的相似度，0代表字符串相等，数字越大表示字符串越不像。 + * + *

+ * 这个算法取自http://www.merriampark.com + * /ld.htm。 它计算的是从字符串1转变到字符串2所需要的删除、插入和替换的步骤数。 + *

+ * + * 
+     * StringUtil.getLevenshteinDistance(null, *)             = IllegalArgumentException
+     * StringUtil.getLevenshteinDistance(*, null)             = IllegalArgumentException
+     * StringUtil.getLevenshteinDistance("","")               = 0
+     * StringUtil.getLevenshteinDistance("","a")              = 1
+     * StringUtil.getLevenshteinDistance("aaapppp", "")       = 7
+     * StringUtil.getLevenshteinDistance("frog", "fog")       = 1
+     * StringUtil.getLevenshteinDistance("fly", "ant")        = 3
+     * StringUtil.getLevenshteinDistance("elephant", "hippo") = 7
+     * StringUtil.getLevenshteinDistance("hippo", "elephant") = 7
+     * StringUtil.getLevenshteinDistance("hippo", "zzzzzzzz") = 8
+     * StringUtil.getLevenshteinDistance("hello", "hallo")    = 1
+     *
+ * + * @param s + * 第一个字符串，如果是null，则看作空字符串 + * @param t + * 第二个字符串，如果是null，则看作空字符串 + * + * @return 相似度值 + */ + public static int getLevenshteinDistance(String s, String t) { + s = defaultIfNull(s); + t = defaultIfNull(t); + + int[][] d; // matrix + int n; // length of s + int m; // length of t + int i; // iterates through s + int j; // iterates through t + char s_i; // ith character of s + char t_j; // jth character of t + int cost; // cost + + // Step 1 + n = s.length(); + m = t.length(); + + if (n == 0) { + return m; + } + + if (m == 0) { + return n; + } + + d = new int[n + 1][m + 1]; + + // Step 2 + for (i = 0; i <= n; i++) { + d[i][0] = i; + } + + for (j = 0; j <= m; j++) { + d[0][j] = j; + } + + // Step 3 + for (i = 1; i <= n; i++) { + s_i = s.charAt(i - 1); + + // Step 4 + for (j = 1; j <= m; j++) { + t_j = t.charAt(j - 1); + + // Step 5 + if (s_i == t_j) { + cost = 0; + } + else { + cost = 1; + } + + // Step 6 + d[i][j] = min(d[i - 1][j] + 1, d[i][j - 1] + 1, d[i - 1][j - 1] + cost); + } + } + + // Step 7 + return d[n][m]; + } + + + /** + * 取得最小数。 + * + * @param a + * 整数1 + * @param b + * 整数2 + * @param c + * 整数3 + * + * @return 三个数中的最小值 + */ + private static int min(int a, int b, int c) { + if (b < a) { + a = b; + } + + if (c < a) { + a = c; + } + + return a; + } +} diff --git a/rocketmq-filtersrv/src/main/java/com/alibaba/common/lang/exception/ChainedError.java b/rocketmq-filtersrv/src/main/java/com/alibaba/common/lang/exception/ChainedError.java new file mode 100644 index 000000000..b15004a59 --- /dev/null +++ b/rocketmq-filtersrv/src/main/java/com/alibaba/common/lang/exception/ChainedError.java @@ -0,0 +1,113 @@ +package com.alibaba.common.lang.exception; + +import java.io.PrintStream; +import java.io.PrintWriter; + + +/** + * 可嵌套的异常. + * + * @author Michael Zhou + * @version $Id: ChainedError.java 1291 2005-03-04 03:23:30Z baobao $ + */ +public class ChainedError extends Error implements ChainedThrowable { + private static final long serialVersionUID = 3762250833760368436L; + private final ChainedThrowable delegate = new ChainedThrowableDelegate(this); + private Throwable cause; + + + /** + * 构造一个空的异常. + */ + public ChainedError() { + super(); + } + + + /** + * 构造一个异常, 指明异常的详细信息. + * + * @param message + * 详细信息 + */ + public ChainedError(String message) { + super(message); + } + + + /** + * 构造一个异常, 指明引起这个异常的起因. + * + * @param cause + * 异常的起因 + */ + public ChainedError(Throwable cause) { + super((cause == null) ? null : cause.getMessage()); + this.cause = cause; + } + + + /** + * 构造一个异常, 指明引起这个异常的起因. + * + * @param message + * 详细信息 + * @param cause + * 异常的起因 + */ + public ChainedError(String message, Throwable cause) { + super(message); + this.cause = cause; + } + + + /** + * 取得引起这个异常的起因. + * + * @return 异常的起因. + */ + public Throwable getCause() { + return cause; + } + + + /** + * 打印调用栈到标准错误. + */ + public void printStackTrace() { + delegate.printStackTrace(); + } + + + /** + * 打印调用栈到指定输出流. + * + * @param stream + * 输出字节流. + */ + public void printStackTrace(PrintStream stream) { + delegate.printStackTrace(stream); + } + + + /** + * 打印调用栈到指定输出流. + * + * @param writer + * 输出字符流. + */ + public void printStackTrace(PrintWriter writer) { + delegate.printStackTrace(writer); + } + + + /** + * 打印异常的调用栈, 不包括起因异常的信息. + * + * @param writer + * 打印到输出流 + */ + public void printCurrentStackTrace(PrintWriter writer) { + super.printStackTrace(writer); + } +} diff --git a/rocketmq-filtersrv/src/main/java/com/alibaba/common/lang/exception/ChainedException.java b/rocketmq-filtersrv/src/main/java/com/alibaba/common/lang/exception/ChainedException.java new file mode 100644 index 000000000..f5149999b --- /dev/null +++ b/rocketmq-filtersrv/src/main/java/com/alibaba/common/lang/exception/ChainedException.java @@ -0,0 +1,113 @@ +package com.alibaba.common.lang.exception; + +import java.io.PrintStream; +import java.io.PrintWriter; + + +/** + * 可嵌套的异常. + * + * @author Michael Zhou + * @version $Id: ChainedException.java 1291 2005-03-04 03:23:30Z baobao $ + */ +public class ChainedException extends Exception implements ChainedThrowable { + private static final long serialVersionUID = 3257568390900888633L; + private final ChainedThrowable delegate = new ChainedThrowableDelegate(this); + private Throwable cause; + + + /** + * 构造一个空的异常. + */ + public ChainedException() { + super(); + } + + + /** + * 构造一个异常, 指明异常的详细信息. + * + * @param message + * 详细信息 + */ + public ChainedException(String message) { + super(message); + } + + + /** + * 构造一个异常, 指明引起这个异常的起因. + * + * @param cause + * 异常的起因 + */ + public ChainedException(Throwable cause) { + super((cause == null) ? null : cause.getMessage()); + this.cause = cause; + } + + + /** + * 构造一个异常, 指明引起这个异常的起因. + * + * @param message + * 详细信息 + * @param cause + * 异常的起因 + */ + public ChainedException(String message, Throwable cause) { + super(message); + this.cause = cause; + } + + + /** + * 取得引起这个异常的起因. + * + * @return 异常的起因. + */ + public Throwable getCause() { + return cause; + } + + + /** + * 打印调用栈到标准错误. + */ + public void printStackTrace() { + delegate.printStackTrace(); + } + + + /** + * 打印调用栈到指定输出流. + * + * @param stream + * 输出字节流. + */ + public void printStackTrace(PrintStream stream) { + delegate.printStackTrace(stream); + } + + + /** + * 打印调用栈到指定输出流. + * + * @param writer + * 输出字符流. + */ + public void printStackTrace(PrintWriter writer) { + delegate.printStackTrace(writer); + } + + + /** + * 打印异常的调用栈, 不包括起因异常的信息. + * + * @param writer + * 打印到输出流 + */ + public void printCurrentStackTrace(PrintWriter writer) { + super.printStackTrace(writer); + } +} diff --git a/rocketmq-filtersrv/src/main/java/com/alibaba/common/lang/exception/ChainedRuntimeException.java b/rocketmq-filtersrv/src/main/java/com/alibaba/common/lang/exception/ChainedRuntimeException.java new file mode 100644 index 000000000..6b396b767 --- /dev/null +++ b/rocketmq-filtersrv/src/main/java/com/alibaba/common/lang/exception/ChainedRuntimeException.java @@ -0,0 +1,113 @@ +package com.alibaba.common.lang.exception; + +import java.io.PrintStream; +import java.io.PrintWriter; + + +/** + * 可嵌套的异常. + * + * @author Michael Zhou + * @version $Id: ChainedRuntimeException.java 1291 2005-03-04 03:23:30Z baobao $ + */ +public class ChainedRuntimeException extends RuntimeException implements ChainedThrowable { + private static final long serialVersionUID = 3257005466717533235L; + private final ChainedThrowable delegate = new ChainedThrowableDelegate(this); + private Throwable cause; + + + /** + * 构造一个空的异常. + */ + public ChainedRuntimeException() { + super(); + } + + + /** + * 构造一个异常, 指明异常的详细信息. + * + * @param message + * 详细信息 + */ + public ChainedRuntimeException(String message) { + super(message); + } + + + /** + * 构造一个异常, 指明引起这个异常的起因. + * + * @param cause + * 异常的起因 + */ + public ChainedRuntimeException(Throwable cause) { + super((cause == null) ? null : cause.getMessage()); + this.cause = cause; + } + + + /** + * 构造一个异常, 指明引起这个异常的起因. + * + * @param message + * 详细信息 + * @param cause + * 异常的起因 + */ + public ChainedRuntimeException(String message, Throwable cause) { + super(message); + this.cause = cause; + } + + + /** + * 取得引起这个异常的起因. + * + * @return 异常的起因. + */ + public Throwable getCause() { + return cause; + } + + + /** + * 打印调用栈到标准错误. + */ + public void printStackTrace() { + delegate.printStackTrace(); + } + + + /** + * 打印调用栈到指定输出流. + * + * @param stream + * 输出字节流. + */ + public void printStackTrace(PrintStream stream) { + delegate.printStackTrace(stream); + } + + + /** + * 打印调用栈到指定输出流. + * + * @param writer + * 输出字符流. + */ + public void printStackTrace(PrintWriter writer) { + delegate.printStackTrace(writer); + } + + + /** + * 打印异常的调用栈, 不包括起因异常的信息. + * + * @param writer + * 打印到输出流 + */ + public void printCurrentStackTrace(PrintWriter writer) { + super.printStackTrace(writer); + } +} diff --git a/rocketmq-filtersrv/src/main/java/com/alibaba/common/lang/exception/ChainedThrowable.java b/rocketmq-filtersrv/src/main/java/com/alibaba/common/lang/exception/ChainedThrowable.java new file mode 100644 index 000000000..d02370c1e --- /dev/null +++ b/rocketmq-filtersrv/src/main/java/com/alibaba/common/lang/exception/ChainedThrowable.java @@ -0,0 +1,54 @@ +package com.alibaba.common.lang.exception; + +import java.io.PrintStream; +import java.io.PrintWriter; +import java.io.Serializable; + + +/** + * 实现此接口的异常, 是由另一个异常引起的. + * + * @author Michael Zhou + * @version $Id: ChainedThrowable.java 509 2004-02-16 05:42:07Z baobao $ + */ +public interface ChainedThrowable extends Serializable { + /** + * 取得异常的起因. + * + * @return 异常的起因. + */ + Throwable getCause(); + + + /** + * 打印调用栈到标准错误. + */ + void printStackTrace(); + + + /** + * 打印调用栈到指定输出流. + * + * @param stream + * 输出字节流. + */ + void printStackTrace(PrintStream stream); + + + /** + * 打印调用栈到指定输出流. + * + * @param writer + * 输出字符流. + */ + void printStackTrace(PrintWriter writer); + + + /** + * 打印异常的调用栈, 不包括起因异常的信息. + * + * @param writer + * 打印到输出流 + */ + void printCurrentStackTrace(PrintWriter writer); +} diff --git a/rocketmq-filtersrv/src/main/java/com/alibaba/common/lang/exception/ChainedThrowableDelegate.java b/rocketmq-filtersrv/src/main/java/com/alibaba/common/lang/exception/ChainedThrowableDelegate.java new file mode 100644 index 000000000..a3b88618a --- /dev/null +++ b/rocketmq-filtersrv/src/main/java/com/alibaba/common/lang/exception/ChainedThrowableDelegate.java @@ -0,0 +1,252 @@ +package com.alibaba.common.lang.exception; + +import java.io.PrintStream; +import java.io.PrintWriter; +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.rmi.RemoteException; +import java.sql.SQLException; + + +/** + * 可嵌套的异常代理, 简化可嵌套的异常的实现. + * + * @author Michael Zhou + * @version $Id: ChainedThrowableDelegate.java 1291 2005-03-04 03:23:30Z baobao + * $ + */ +public class ChainedThrowableDelegate implements ChainedThrowable { + private static final long serialVersionUID = 3257288032683241523L; + + /** 表示异常不存在的常量. */ + protected static final Throwable NO_CAUSE = new Throwable(); + + /** 常见的用来取得异常起因的方法名. */ + private static final String[] CAUSE_METHOD_NAMES = { "getNested", "getNestedException", + "getNextException", "getTargetException", + "getException", "getSourceException", + "getCausedByException", "getRootCause", "getCause" }; + + /** 常见的用来取得异常起因的字段名. */ + private static final String[] CAUSE_FIELD_NAMES = { "detail" }; + + /** 被代理的Throwable对象. */ + protected Throwable delegatedThrowable; + + + /** + * 创建一个Throwable代理. + * + * @param throwable + * 被代理的异常 + */ + public ChainedThrowableDelegate(Throwable throwable) { + this.delegatedThrowable = throwable; + } + + + /** + * 取得被代理的异常的起因. + * + * @return 异常的起因. + */ + public Throwable getCause() { + Throwable cause = getCauseByWellKnownTypes(delegatedThrowable); + + for (Class throwableClass = delegatedThrowable.getClass(); (cause == null) + && Throwable.class.isAssignableFrom(throwableClass); throwableClass = + throwableClass.getSuperclass()) { + // 尝试常见的方法 + for (int i = 0; (cause == null) && (i < CAUSE_METHOD_NAMES.length); i++) { + cause = getCauseByMethodName(delegatedThrowable, throwableClass, CAUSE_METHOD_NAMES[i]); + } + + // 尝试常见的字段 + for (int i = 0; (cause == null) && (i < CAUSE_FIELD_NAMES.length); i++) { + cause = getCauseByFieldName(delegatedThrowable, throwableClass, CAUSE_FIELD_NAMES[i]); + } + } + + if (cause == delegatedThrowable) { + cause = null; + } + + if (cause == NO_CAUSE) { + return null; + } + + return cause; + } + + + /** + * 取得常见Throwable类的异常起因. + * + * @param throwable + * 异常 + * + * @return 异常起因 + */ + protected Throwable getCauseByWellKnownTypes(Throwable throwable) { + Throwable cause = null; + boolean isWellKnownType = false; + + if (throwable instanceof ChainedThrowable) { + isWellKnownType = true; + cause = ((ChainedThrowable) throwable).getCause(); + } + else if (throwable instanceof SQLException) { + isWellKnownType = true; + cause = ((SQLException) throwable).getNextException(); + } + else if (throwable instanceof InvocationTargetException) { + isWellKnownType = true; + cause = ((InvocationTargetException) throwable).getTargetException(); + } + else if (throwable instanceof RemoteException) { + isWellKnownType = true; + cause = ((RemoteException) throwable).detail; + } + + if (isWellKnownType && (cause == null)) { + return NO_CAUSE; + } + + return cause; + } + + + /** + * 通过常见的方法动态地取得异常起因. + * + * @param throwable + * 异常 + * @param throwableClass + * 异常类 + * @param methodName + * 方法名 + * + * @return 异常起因或NO_CAUSE + */ + protected Throwable getCauseByMethodName(Throwable throwable, Class throwableClass, String methodName) { + Method method = null; + + try { + method = throwableClass.getMethod(methodName, new Class[0]); + } + catch (NoSuchMethodException ignored) { + } + + if ((method != null) && Throwable.class.isAssignableFrom(method.getReturnType())) { + Throwable cause = null; + + try { + cause = (Throwable) method.invoke(throwable, new Object[0]); + } + catch (IllegalAccessException ignored) { + } + catch (IllegalArgumentException ignored) { + } + catch (InvocationTargetException ignored) { + } + + if (cause == null) { + return NO_CAUSE; + } + + return cause; + } + + return null; + } + + + /** + * 通过常见的方法动态地取得异常起因. + * + * @param throwable + * 异常 + * @param throwableClass + * 异常类 + * @param fieldName + * 字段名 + * + * @return 异常起因或NO_CAUSE + */ + protected Throwable getCauseByFieldName(Throwable throwable, Class throwableClass, String fieldName) { + Field field = null; + + try { + field = throwableClass.getField(fieldName); + } + catch (NoSuchFieldException ignored) { + } + + if ((field != null) && Throwable.class.isAssignableFrom(field.getType())) { + Throwable cause = null; + + try { + cause = (Throwable) field.get(throwable); + } + catch (IllegalAccessException ignored) { + } + catch (IllegalArgumentException ignored) { + } + + if (cause == null) { + return NO_CAUSE; + } + + return cause; + } + + return null; + } + + + /** + * 打印调用栈到标准错误. + */ + public void printStackTrace() { + ExceptionHelper.printStackTrace(this); + } + + + /** + * 打印调用栈到指定输出流. + * + * @param stream + * 输出字节流. + */ + public void printStackTrace(PrintStream stream) { + ExceptionHelper.printStackTrace(this, stream); + } + + + /** + * 打印调用栈到指定输出流. + * + * @param writer + * 输出字符流. + */ + public void printStackTrace(PrintWriter writer) { + ExceptionHelper.printStackTrace(this, writer); + } + + + /** + * 打印异常的调用栈, 不包括起因异常的信息. + * + * @param writer + * 打印到输出流 + */ + public void printCurrentStackTrace(PrintWriter writer) { + if (delegatedThrowable instanceof ChainedThrowable) { + ((ChainedThrowable) delegatedThrowable).printCurrentStackTrace(writer); + } + else { + delegatedThrowable.printStackTrace(writer); + } + } +} diff --git a/rocketmq-filtersrv/src/main/java/com/alibaba/common/lang/exception/ExceptionHelper.java b/rocketmq-filtersrv/src/main/java/com/alibaba/common/lang/exception/ExceptionHelper.java new file mode 100644 index 000000000..24d4d3b1d --- /dev/null +++ b/rocketmq-filtersrv/src/main/java/com/alibaba/common/lang/exception/ExceptionHelper.java @@ -0,0 +1,418 @@ +package com.alibaba.common.lang.exception; + +import java.io.PrintStream; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.List; + + +/** + * 异常的辅助类. + * + * @author Michael Zhou + * @version $Id: ExceptionHelper.java 643 2004-03-07 07:29:35Z baobao $ + */ +public class ExceptionHelper { + private static final String STRING_EXCEPTION_MESSAGE = ": "; + private static final String STRING_CAUSED_BY = "Caused by: "; + private static final String STRING_MORE_PREFIX = "\t... "; + private static final String STRING_MORE_SUFFIX = " more"; + private static final String STRING_STACK_TRACE_PREFIX = "\tat "; + private static final String STRING_CR = "\r"; + private static final String STRING_LF = "\n"; + private static final String GET_STACK_TRACE_METHOD_NAME = "getStackTrace"; + private static Method GET_STACK_TRACE_METHOD; + + static { + // JDK1.4支持Throwable.getStackTrace()方法 + try { + GET_STACK_TRACE_METHOD = Throwable.class.getMethod(GET_STACK_TRACE_METHOD_NAME, new Class[0]); + } + catch (NoSuchMethodException e) { + } + } + + + /** + * 从ChainedThrowable实例中取得Throwable对象. + * + * @param throwable + * ChainedThrowable实例 + * + * @return Throwable对象 + */ + private static Throwable getThrowable(ChainedThrowable throwable) { + if (throwable instanceof ChainedThrowableDelegate) { + return ((ChainedThrowableDelegate) throwable).delegatedThrowable; + } + else { + return (Throwable) throwable; + } + } + + + /** + * 将Throwable转换成ChainedThrowable. 如果已经是 + * ChainedThrowable了, 则直接返回, 否则将它包装在 + * ChainedThrowableDelegate中返回. + * + * @param throwable + * Throwable对象 + * + * @return ChainedThrowable对象 + */ + public static ChainedThrowable getChainedThrowable(Throwable throwable) { + if ((throwable != null) && !(throwable instanceof ChainedThrowable)) { + return new ChainedThrowableDelegate(throwable); + } + + return (ChainedThrowable) throwable; + } + + + /** + * 取得被代理的异常的起因, 如果起因不是ChainedThrowable, 则用 + * ChainedThrowableDelegate包装并返回. + * + * @param throwable + * 异常 + * + * @return 异常的起因 + */ + public static ChainedThrowable getChainedThrowableCause(ChainedThrowable throwable) { + return getChainedThrowable(throwable.getCause()); + } + + + /** + * 打印调用栈到标准错误. + * + * @param throwable + * 异常 + */ + public static void printStackTrace(ChainedThrowable throwable) { + printStackTrace(throwable, System.err); + } + + + /** + * 打印调用栈到指定输出流. + * + * @param throwable + * 异常 + * @param stream + * 输出字节流 + */ + public static void printStackTrace(ChainedThrowable throwable, PrintStream stream) { + printStackTrace(throwable, new PrintWriter(stream)); + } + + + /** + * 打印调用栈到指定输出流. + * + * @param throwable + * 异常 + * @param writer + * 输出字符流 + */ + public static void printStackTrace(ChainedThrowable throwable, PrintWriter writer) { + synchronized (writer) { + String[] currentStack = analyzeStackTrace(throwable); + + printThrowableMessage(throwable, writer, false); + + for (int i = 0; i < currentStack.length; i++) { + writer.println(currentStack[i]); + } + + printStackTraceRecursive(throwable, writer, currentStack); + + writer.flush(); + } + } + + + /** + * 递归地打印所有异常链的调用栈. + * + * @param throwable + * 异常 + * @param writer + * 输出流 + * @param currentStack + * 当前的堆栈 + */ + private static void printStackTraceRecursive(ChainedThrowable throwable, PrintWriter writer, + String[] currentStack) { + ChainedThrowable cause = getChainedThrowableCause(throwable); + + if (cause != null) { + String[] causeStack = analyzeStackTrace(cause); + int i = currentStack.length - 1; + int j = causeStack.length - 1; + + for (; (i >= 0) && (j >= 0); i--, j--) { + if (!currentStack[i].equals(causeStack[j])) { + break; + } + } + + printThrowableMessage(cause, writer, true); + + for (i = 0; i <= j; i++) { + writer.println(causeStack[i]); + } + + if (j < (causeStack.length - 1)) { + writer.println(STRING_MORE_PREFIX + (causeStack.length - j - 1) + STRING_MORE_SUFFIX); + } + + printStackTraceRecursive(cause, writer, causeStack); + } + } + + + /** + * 打印异常的message. + * + * @param throwable + * 异常 + * @param writer + * 输出流 + * @param cause + * 是否是起因异常 + */ + private static void printThrowableMessage(ChainedThrowable throwable, PrintWriter writer, boolean cause) { + StringBuffer buffer = new StringBuffer(); + + if (cause) { + buffer.append(STRING_CAUSED_BY); + } + + Throwable t = getThrowable(throwable); + + buffer.append(t.getClass().getName()); + + String message = t.getMessage(); + + if (message != null) { + buffer.append(STRING_EXCEPTION_MESSAGE).append(message); + } + + writer.println(buffer.toString()); + } + + + /** + * 分析异常的调用栈, 取得当前异常的信息, 不包括起因异常的信息. + * + * @param throwable + * 取得指定异常的调用栈 + * + * @return 调用栈数组 + */ + private static String[] analyzeStackTrace(ChainedThrowable throwable) { + if (GET_STACK_TRACE_METHOD != null) { + Throwable t = getThrowable(throwable); + + try { + Object[] stackTrace = (Object[]) GET_STACK_TRACE_METHOD.invoke(t, new Object[0]); + String[] list = new String[stackTrace.length]; + + for (int i = 0; i < stackTrace.length; i++) { + list[i] = STRING_STACK_TRACE_PREFIX + stackTrace[i]; + } + + return list; + } + catch (IllegalAccessException e) { + } + catch (IllegalArgumentException e) { + } + catch (InvocationTargetException e) { + } + } + + return new StackTraceAnalyzer(throwable).getLines(); + } + + /** + * 分析stack trace的辅助类. + */ + private static class StackTraceAnalyzer { + private Throwable throwable; + private String message; + private StackTraceEntry currentEntry = new StackTraceEntry(); + private StackTraceEntry selectedEntry = currentEntry; + private StackTraceEntry entry; + + + StackTraceAnalyzer(ChainedThrowable throwable) { + this.throwable = getThrowable(throwable); + this.message = this.throwable.getMessage(); + + // 取得stack trace字符串. + StringWriter writer = new StringWriter(); + PrintWriter pw = new PrintWriter(writer); + + throwable.printCurrentStackTrace(pw); + + String stackTraceDump = writer.toString(); + + // 分割字符串, 按行分割, 但不能割开message字串 + int p = 0; + int i = -1; + int j = -1; + int k = -1; + + while (p < stackTraceDump.length()) { + boolean includesMessage = false; + int s = p; + + if ((i == -1) && (message != null)) { + i = stackTraceDump.indexOf(message, p); + } + + if (j == -1) { + j = stackTraceDump.indexOf(STRING_CR, p); + } + + if (k == -1) { + k = stackTraceDump.indexOf(STRING_LF, p); + } + + // 如果找到message + if ((i != -1) && ((j == -1) || (i <= j)) && ((k == -1) || (i <= k))) { + includesMessage = true; + p = i + message.length(); + i = -1; + + if (j < p) { + j = -1; + } + + if (k < p) { + k = -1; + } + + // 继续直到换行 + } + + // 如果找到换行CR或CRLF + if ((j != -1) && ((k == -1) || (j < k))) { + p = j + 1; + + if ((p < stackTraceDump.length()) && (stackTraceDump.charAt(p) == STRING_LF.charAt(0))) { + p++; // CRLF + } + + addLine(stackTraceDump.substring(s, j), includesMessage); + + j = -1; + + if (k < p) { + k = -1; + } + + continue; + } + + // 如果找到LF + if (k != -1) { + int q = k + 1; + + addLine(stackTraceDump.substring(s, k), includesMessage); + p = q; + k = -1; + continue; + } + + // 如果都没找到, 说明到了最后一行 + int q = stackTraceDump.length(); + + if ((p + 1) < q) { + addLine(stackTraceDump.substring(s), includesMessage); + p = q; + } + } + + // 选择"较小"的entry + if (currentEntry.compareTo(selectedEntry) < 0) { + selectedEntry = currentEntry; + } + } + + + private void addLine(String line, boolean includesMessage) { + StackTraceEntry nextEntry = currentEntry.accept(line, includesMessage); + + if (nextEntry != null) { + // 选择"较小"的entry + if (currentEntry.compareTo(selectedEntry) < 0) { + selectedEntry = currentEntry; + } + + currentEntry = nextEntry; + } + } + + + String[] getLines() { + return (String[]) selectedEntry.lines.toArray(new String[selectedEntry.lines.size()]); + } + + private class StackTraceEntry implements Comparable { + private List lines = new ArrayList(10); + private int includesMessage = 0; + private int includesThrowable = 0; + private int count = 0; + + + StackTraceEntry accept(String line, boolean includesMessage) { + // 如果是...at XXX.java(Line...), 则加入到lines列表中. + // 否则创建并返回新的entry. + if (line.startsWith(STRING_STACK_TRACE_PREFIX)) { + lines.add(line); + count++; + return null; + } + else if (count > 0) { + StackTraceEntry newEntry = new StackTraceEntry(); + + newEntry.accept(line, includesMessage); + return newEntry; + } + + // 设置权重 + if (includesMessage) { + this.includesMessage = 1; + } + + if (line.indexOf(throwable.getClass().getName()) != -1) { + this.includesThrowable = 1; + } + + return null; + } + + + public int compareTo(Object o) { + StackTraceEntry otherEntry = (StackTraceEntry) o; + int thisWeight = includesMessage + includesThrowable; + int otherWeight = otherEntry.includesMessage + otherEntry.includesThrowable; + + // weight大的排在前, 如果weight相同, 则count小的排在前 + if (thisWeight == otherWeight) { + return count - otherEntry.count; + } + else { + return otherWeight - thisWeight; + } + } + } + } +} diff --git a/rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/FilterServerOuterAPI.java b/rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/FilterServerOuterAPI.java new file mode 100644 index 000000000..2e7230130 --- /dev/null +++ b/rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/FilterServerOuterAPI.java @@ -0,0 +1,84 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.filtersrv; + +import com.alibaba.rocketmq.client.exception.MQBrokerException; +import com.alibaba.rocketmq.common.protocol.RequestCode; +import com.alibaba.rocketmq.common.protocol.ResponseCode; +import com.alibaba.rocketmq.common.protocol.header.filtersrv.RegisterFilterServerRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.filtersrv.RegisterFilterServerResponseHeader; +import com.alibaba.rocketmq.remoting.RemotingClient; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; +import com.alibaba.rocketmq.remoting.exception.RemotingConnectException; +import com.alibaba.rocketmq.remoting.exception.RemotingSendRequestException; +import com.alibaba.rocketmq.remoting.exception.RemotingTimeoutException; +import com.alibaba.rocketmq.remoting.netty.NettyClientConfig; +import com.alibaba.rocketmq.remoting.netty.NettyRemotingClient; +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; + + +/** + * Broker对外调用的API封装 + * + * @author shijia.wxr + * @since 2014-4-10 + */ +public class FilterServerOuterAPI { + private final RemotingClient remotingClient; + + + public FilterServerOuterAPI() { + this.remotingClient = new NettyRemotingClient(new NettyClientConfig()); + } + + + public void start() { + this.remotingClient.start(); + } + + + public void shutdown() { + this.remotingClient.shutdown(); + } + + + public RegisterFilterServerResponseHeader registerFilterServerToBroker(// + final String brokerAddr,// 1 + final String filterServerAddr// 2 + ) throws RemotingCommandException, RemotingConnectException, RemotingSendRequestException, + RemotingTimeoutException, InterruptedException, MQBrokerException { + RegisterFilterServerRequestHeader requestHeader = new RegisterFilterServerRequestHeader(); + requestHeader.setFilterServerAddr(filterServerAddr); + RemotingCommand request = + RemotingCommand.createRequestCommand(RequestCode.REGISTER_FILTER_SERVER, requestHeader); + + RemotingCommand response = this.remotingClient.invokeSync(brokerAddr, request, 3000); + assert response != null; + switch (response.getCode()) { + case ResponseCode.SUCCESS: { + RegisterFilterServerResponseHeader responseHeader = + (RegisterFilterServerResponseHeader) response + .decodeCommandCustomHeader(RegisterFilterServerResponseHeader.class); + + return responseHeader; + } + default: + break; + } + + throw new MQBrokerException(response.getCode(), response.getRemark()); + } +} diff --git a/rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/FiltersrvConfig.java b/rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/FiltersrvConfig.java new file mode 100644 index 000000000..ff94ed2a2 --- /dev/null +++ b/rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/FiltersrvConfig.java @@ -0,0 +1,144 @@ +package com.alibaba.rocketmq.filtersrv; + +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.annotation.ImportantField; +import com.alibaba.rocketmq.remoting.common.RemotingUtil; + + +public class FiltersrvConfig { + private String rocketmqHome = System.getProperty(MixAll.ROCKETMQ_HOME_PROPERTY, + System.getenv(MixAll.ROCKETMQ_HOME_ENV)); + + @ImportantField + private String namesrvAddr = System.getProperty(MixAll.NAMESRV_ADDR_PROPERTY, + System.getenv(MixAll.NAMESRV_ADDR_ENV)); + + // 连接到哪个Broker + private String connectWhichBroker = "127.0.0.1:10911"; + // Filter Server对外服务的IP + private String filterServerIP = RemotingUtil.getLocalAddress(); + // 消息超过指定大小，开始压缩 + private int compressMsgBodyOverHowmuch = 1024 * 8; + // 压缩Level + private int zipCompressLevel = 5; + + // 是否允许客户端上传Java类 + private boolean clientUploadFilterClassEnable = true; + + // 过滤类的仓库地址 + private String filterClassRepertoryUrl = "http://fsrep.tbsite.net/filterclass"; + + private int fsServerAsyncSemaphoreValue = 2048; + private int fsServerCallbackExecutorThreads = 64; + private int fsServerWorkerThreads = 64; + + + public String getRocketmqHome() { + return rocketmqHome; + } + + + public void setRocketmqHome(String rocketmqHome) { + this.rocketmqHome = rocketmqHome; + } + + + public String getNamesrvAddr() { + return namesrvAddr; + } + + + public void setNamesrvAddr(String namesrvAddr) { + this.namesrvAddr = namesrvAddr; + } + + + public String getConnectWhichBroker() { + return connectWhichBroker; + } + + + public void setConnectWhichBroker(String connectWhichBroker) { + this.connectWhichBroker = connectWhichBroker; + } + + + public String getFilterServerIP() { + return filterServerIP; + } + + + public void setFilterServerIP(String filterServerIP) { + this.filterServerIP = filterServerIP; + } + + + public int getCompressMsgBodyOverHowmuch() { + return compressMsgBodyOverHowmuch; + } + + + public void setCompressMsgBodyOverHowmuch(int compressMsgBodyOverHowmuch) { + this.compressMsgBodyOverHowmuch = compressMsgBodyOverHowmuch; + } + + + public int getZipCompressLevel() { + return zipCompressLevel; + } + + + public void setZipCompressLevel(int zipCompressLevel) { + this.zipCompressLevel = zipCompressLevel; + } + + + public boolean isClientUploadFilterClassEnable() { + return clientUploadFilterClassEnable; + } + + + public void setClientUploadFilterClassEnable(boolean clientUploadFilterClassEnable) { + this.clientUploadFilterClassEnable = clientUploadFilterClassEnable; + } + + + public String getFilterClassRepertoryUrl() { + return filterClassRepertoryUrl; + } + + + public void setFilterClassRepertoryUrl(String filterClassRepertoryUrl) { + this.filterClassRepertoryUrl = filterClassRepertoryUrl; + } + + + public int getFsServerAsyncSemaphoreValue() { + return fsServerAsyncSemaphoreValue; + } + + + public void setFsServerAsyncSemaphoreValue(int fsServerAsyncSemaphoreValue) { + this.fsServerAsyncSemaphoreValue = fsServerAsyncSemaphoreValue; + } + + + public int getFsServerCallbackExecutorThreads() { + return fsServerCallbackExecutorThreads; + } + + + public void setFsServerCallbackExecutorThreads(int fsServerCallbackExecutorThreads) { + this.fsServerCallbackExecutorThreads = fsServerCallbackExecutorThreads; + } + + + public int getFsServerWorkerThreads() { + return fsServerWorkerThreads; + } + + + public void setFsServerWorkerThreads(int fsServerWorkerThreads) { + this.fsServerWorkerThreads = fsServerWorkerThreads; + } +} diff --git a/rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/FiltersrvController.java b/rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/FiltersrvController.java new file mode 100644 index 000000000..bc4746171 --- /dev/null +++ b/rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/FiltersrvController.java @@ -0,0 +1,242 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.filtersrv; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.rocketmq.client.consumer.DefaultMQPullConsumer; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.ThreadFactoryImpl; +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.common.protocol.header.filtersrv.RegisterFilterServerResponseHeader; +import com.alibaba.rocketmq.filtersrv.filter.FilterClassManager; +import com.alibaba.rocketmq.filtersrv.processor.DefaultRequestProcessor; +import com.alibaba.rocketmq.filtersrv.stats.FilterServerStatsManager; +import com.alibaba.rocketmq.remoting.RemotingServer; +import com.alibaba.rocketmq.remoting.netty.NettyRemotingServer; +import com.alibaba.rocketmq.remoting.netty.NettyServerConfig; + + +/** + * Filter Server服务控制 + * + * @author shijia.wxr + * @since 2014-4-10 + */ +public class FiltersrvController { + private static final Logger log = LoggerFactory.getLogger(LoggerName.FiltersrvLoggerName); + // Filter Server配置 + private final FiltersrvConfig filtersrvConfig; + // 通信层配置 + private final NettyServerConfig nettyServerConfig; + // 服务端通信层对象 + private RemotingServer remotingServer; + // 服务端网络请求处理线程池 + private ExecutorService remotingExecutor; + + private final FilterClassManager filterClassManager; + + // 访问Broker的api封装 + private final FilterServerOuterAPI filterServerOuterAPI = new FilterServerOuterAPI(); + + private final DefaultMQPullConsumer defaultMQPullConsumer = new DefaultMQPullConsumer( + MixAll.FILTERSRV_CONSUMER_GROUP); + + private volatile String brokerName = null; + + // 定时线程 + private final ScheduledExecutorService scheduledExecutorService = Executors + .newSingleThreadScheduledExecutor(new ThreadFactoryImpl("FSScheduledThread")); + + private final FilterServerStatsManager filterServerStatsManager = new FilterServerStatsManager(); + + + public FiltersrvController(FiltersrvConfig filtersrvConfig, NettyServerConfig nettyServerConfig) { + this.filtersrvConfig = filtersrvConfig; + this.nettyServerConfig = nettyServerConfig; + this.filterClassManager = new FilterClassManager(this); + } + + + public boolean initialize() { + // 打印服务器配置参数 + MixAll.printObjectProperties(log, this.filtersrvConfig); + + // 初始化通信层 + this.remotingServer = new NettyRemotingServer(this.nettyServerConfig); + + // 初始化线程池 + this.remotingExecutor = + Executors.newFixedThreadPool(nettyServerConfig.getServerWorkerThreads(), + new ThreadFactoryImpl("RemotingExecutorThread_")); + + this.registerProcessor(); + + // 定时向Broker注册自己 + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + + @Override + public void run() { + FiltersrvController.this.registerFilterServerToBroker(); + } + }, 3, 10, TimeUnit.SECONDS); + + // 初始化PullConsumer参数，要比默认参数小。 + this.defaultMQPullConsumer.setBrokerSuspendMaxTimeMillis(this.defaultMQPullConsumer + .getBrokerSuspendMaxTimeMillis() - 1000); + this.defaultMQPullConsumer.setConsumerTimeoutMillisWhenSuspend(this.defaultMQPullConsumer + .getConsumerTimeoutMillisWhenSuspend() - 1000); + + this.defaultMQPullConsumer.setNamesrvAddr(this.filtersrvConfig.getNamesrvAddr()); + this.defaultMQPullConsumer.setInstanceName(String.valueOf(UtilAll.getPid())); + + return true; + } + + + public String localAddr() { + return String.format("%s:%d", this.filtersrvConfig.getFilterServerIP(), + this.remotingServer.localListenPort()); + } + + + public void registerFilterServerToBroker() { + try { + RegisterFilterServerResponseHeader responseHeader = + this.filterServerOuterAPI.registerFilterServerToBroker( + this.filtersrvConfig.getConnectWhichBroker(), this.localAddr()); + this.defaultMQPullConsumer.getDefaultMQPullConsumerImpl().getPullAPIWrapper() + .setDefaultBrokerId(responseHeader.getBrokerId()); + + if (null == this.brokerName) { + this.brokerName = responseHeader.getBrokerName(); + } + + log.info("register filter server<{}> to broker<{}> OK, Return: {} {}", // + this.localAddr(),// + this.filtersrvConfig.getConnectWhichBroker(),// + responseHeader.getBrokerName(),// + responseHeader.getBrokerId()// + ); + } + catch (Exception e) { + log.warn("register filter server Exception", e); + // 如果失败，尝试自杀 + log.warn("access broker failed, kill oneself"); + System.exit(-1); + } + } + + + private void registerProcessor() { + this.remotingServer + .registerDefaultProcessor(new DefaultRequestProcessor(this), this.remotingExecutor); + } + + + public void start() throws Exception { + this.defaultMQPullConsumer.start(); + this.remotingServer.start(); + this.filterServerOuterAPI.start(); + this.defaultMQPullConsumer.getDefaultMQPullConsumerImpl().getPullAPIWrapper() + .setConnectBrokerByUser(true); + this.filterClassManager.start(); + this.filterServerStatsManager.start(); + } + + + public void shutdown() { + this.remotingServer.shutdown(); + this.remotingExecutor.shutdown(); + this.scheduledExecutorService.shutdown(); + this.defaultMQPullConsumer.shutdown(); + this.filterServerOuterAPI.shutdown(); + this.filterClassManager.shutdown(); + this.filterServerStatsManager.shutdown(); + } + + + public RemotingServer getRemotingServer() { + return remotingServer; + } + + + public void setRemotingServer(RemotingServer remotingServer) { + this.remotingServer = remotingServer; + } + + + public ExecutorService getRemotingExecutor() { + return remotingExecutor; + } + + + public void setRemotingExecutor(ExecutorService remotingExecutor) { + this.remotingExecutor = remotingExecutor; + } + + + public FiltersrvConfig getFiltersrvConfig() { + return filtersrvConfig; + } + + + public NettyServerConfig getNettyServerConfig() { + return nettyServerConfig; + } + + + public ScheduledExecutorService getScheduledExecutorService() { + return scheduledExecutorService; + } + + + public FilterServerOuterAPI getFilterServerOuterAPI() { + return filterServerOuterAPI; + } + + + public FilterClassManager getFilterClassManager() { + return filterClassManager; + } + + + public DefaultMQPullConsumer getDefaultMQPullConsumer() { + return defaultMQPullConsumer; + } + + + public String getBrokerName() { + return brokerName; + } + + + public void setBrokerName(String brokerName) { + this.brokerName = brokerName; + } + + + public FilterServerStatsManager getFilterServerStatsManager() { + return filterServerStatsManager; + } +} diff --git a/rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/FiltersrvStartup.java b/rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/FiltersrvStartup.java new file mode 100644 index 000000000..c958ef6e8 --- /dev/null +++ b/rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/FiltersrvStartup.java @@ -0,0 +1,203 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.filtersrv; + +import java.io.BufferedInputStream; +import java.io.FileInputStream; +import java.io.InputStream; +import java.util.Properties; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.PosixParser; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import ch.qos.logback.classic.LoggerContext; +import ch.qos.logback.classic.joran.JoranConfigurator; + +import com.alibaba.rocketmq.common.MQVersion; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.remoting.netty.NettyServerConfig; +import com.alibaba.rocketmq.remoting.netty.NettySystemConfig; +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; +import com.alibaba.rocketmq.srvutil.ServerUtil; + + +/** + * Filter server 启动入口 + * + * @author shijia.wxr + * @since 2014-4-10 + */ +public class FiltersrvStartup { + public static Logger log; + + + public static Options buildCommandlineOptions(final Options options) { + Option opt = new Option("c", "configFile", true, "Filter server config properties file"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("p", "printConfigItem", false, "Print all config item"); + opt.setRequired(false); + options.addOption(opt); + + return options; + } + + + public static void main(String[] args) { + start(createController(args)); + } + + + public static FiltersrvController start(FiltersrvController controller) { + // 启动服务 + try { + controller.start(); + } + catch (Exception e) { + e.printStackTrace(); + System.exit(-1); + } + + String tip = "The Filter Server boot success, " + controller.localAddr(); + log.info(tip); + System.out.println(tip); + + return controller; + } + + + public static FiltersrvController createController(String[] args) { + System.setProperty(RemotingCommand.RemotingVersionKey, Integer.toString(MQVersion.CurrentVersion)); + + // Socket发送缓冲区大小 + if (null == System.getProperty(NettySystemConfig.SystemPropertySocketSndbufSize)) { + NettySystemConfig.SocketSndbufSize = 65535; + } + + // Socket接收缓冲区大小 + if (null == System.getProperty(NettySystemConfig.SystemPropertySocketRcvbufSize)) { + NettySystemConfig.SocketRcvbufSize = 1024; + } + + try { + // 解析命令行 + Options options = ServerUtil.buildCommandlineOptions(new Options()); + final CommandLine commandLine = + ServerUtil.parseCmdLine("mqfiltersrv", args, buildCommandlineOptions(options), + new PosixParser()); + if (null == commandLine) { + System.exit(-1); + return null; + } + + // 初始化配置文件 + final FiltersrvConfig filtersrvConfig = new FiltersrvConfig(); + final NettyServerConfig nettyServerConfig = new NettyServerConfig(); + + if (commandLine.hasOption('c')) { + String file = commandLine.getOptionValue('c'); + if (file != null) { + InputStream in = new BufferedInputStream(new FileInputStream(file)); + Properties properties = new Properties(); + properties.load(in); + MixAll.properties2Object(properties, filtersrvConfig); + System.out.println("load config properties file OK, " + file); + in.close(); + + String port = properties.getProperty("listenPort"); + if (port != null) { + filtersrvConfig.setConnectWhichBroker(String.format("127.0.0.1:%s", port)); + } + } + } + + // 强制设置为0，自动分配端口号 + nettyServerConfig.setListenPort(0); + + nettyServerConfig.setServerAsyncSemaphoreValue(filtersrvConfig.getFsServerAsyncSemaphoreValue()); + nettyServerConfig.setServerCallbackExecutorThreads(filtersrvConfig + .getFsServerCallbackExecutorThreads()); + nettyServerConfig.setServerWorkerThreads(filtersrvConfig.getFsServerWorkerThreads()); + + // 打印默认配置 + if (commandLine.hasOption('p')) { + MixAll.printObjectProperties(null, filtersrvConfig); + MixAll.printObjectProperties(null, nettyServerConfig); + System.exit(0); + } + + MixAll.properties2Object(ServerUtil.commandLine2Properties(commandLine), filtersrvConfig); + + if (null == filtersrvConfig.getRocketmqHome()) { + System.out.println("Please set the " + MixAll.ROCKETMQ_HOME_ENV + + " variable in your environment to match the location of the RocketMQ installation"); + System.exit(-2); + } + + // 初始化Logback + LoggerContext lc = (LoggerContext) LoggerFactory.getILoggerFactory(); + JoranConfigurator configurator = new JoranConfigurator(); + configurator.setContext(lc); + lc.reset(); + configurator.doConfigure(filtersrvConfig.getRocketmqHome() + "/conf/logback_filtersrv.xml"); + log = LoggerFactory.getLogger(LoggerName.FiltersrvLoggerName); + + // 初始化服务控制对象 + final FiltersrvController controller = + new FiltersrvController(filtersrvConfig, nettyServerConfig); + boolean initResult = controller.initialize(); + if (!initResult) { + controller.shutdown(); + System.exit(-3); + } + + Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() { + private volatile boolean hasShutdown = false; + private AtomicInteger shutdownTimes = new AtomicInteger(0); + + + @Override + public void run() { + synchronized (this) { + log.info("shutdown hook was invoked, " + this.shutdownTimes.incrementAndGet()); + if (!this.hasShutdown) { + this.hasShutdown = true; + long begineTime = System.currentTimeMillis(); + controller.shutdown(); + long consumingTimeTotal = System.currentTimeMillis() - begineTime; + log.info("shutdown hook over, consuming time total(ms): " + consumingTimeTotal); + } + } + } + }, "ShutdownHook")); + + return controller; + } + catch (Throwable e) { + e.printStackTrace(); + System.exit(-1); + } + + return null; + } +} diff --git a/rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/filter/DynaCode.java b/rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/filter/DynaCode.java new file mode 100644 index 000000000..0da2e725f --- /dev/null +++ b/rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/filter/DynaCode.java @@ -0,0 +1,496 @@ +package com.alibaba.rocketmq.filtersrv.filter; + +import java.io.BufferedWriter; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.OutputStreamWriter; +import java.io.UnsupportedEncodingException; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLClassLoader; +import java.net.URLDecoder; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import javax.tools.JavaCompiler; +import javax.tools.ToolProvider; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.common.lang.ArrayUtil; +import com.alibaba.common.lang.StringUtil; +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.common.filter.FilterAPI; + + +/** + *

+ * description: java动态编译类，创建该类时需要给定编译的java代码的text的code字符串 + * 再调用compileAndLoadClass方法进行编译，调用compileAndLoadClass之前可以通过相关的 SET方法来设置编译的一些参数 + *

+ * + * @{# DynaCode.java Create on Sep 22, 2011 11:34:10 AM + * + * Copyright (c) 2011 by qihao. + * + * @author qihao + * @version 1.0 + */ +public class DynaCode { + private static final Logger logger = LoggerFactory.getLogger(LoggerName.FiltersrvLoggerName); + + private static final String FILE_SP = System.getProperty("file.separator"); + + private static final String LINE_SP = System.getProperty("line.separator"); + + /** + * 生成java文件的路径 + */ + private String sourcePath = System.getProperty("user.home") + FILE_SP + "rocketmq_filter_class" + FILE_SP + + UtilAll.getPid(); + + /** + * 生成class文件的路径 + */ + private String outPutClassPath = sourcePath; + + /** + * 编译使用的生成ClassPath的ClassLoader + */ + private ClassLoader parentClassLoader; + + /** + * java的text代码 + */ + private List codeStrs; + + /** + * 已经加载好的class + */ + private Map/* class */> loadClass; + + /** + * 编译参数,使用的classpath，如果不指定则使用当前给定的 classloder所具有的classpath进行编译 + */ + private String classpath; + + /** + * 编译参数，同javac的bootclasspath + */ + private String bootclasspath; + + /** + * 编译参数，同javac的extdirs + */ + private String extdirs; + + /** + * 编译参数，同javac的encoding + */ + private String encoding = "UTF-8"; + + /** + * 编译参数，同javac的target + */ + private String target; + + + @SuppressWarnings("unchecked") + public DynaCode(String code) { + this(Thread.currentThread().getContextClassLoader(), ArrayUtil.toList(new String[] { code })); + } + + + public DynaCode(List codeStrs) { + this(Thread.currentThread().getContextClassLoader(), codeStrs); + } + + + public DynaCode(ClassLoader parentClassLoader, List codeStrs) { + this(extractClasspath(parentClassLoader), parentClassLoader, codeStrs); + } + + + public DynaCode(String classpath, ClassLoader parentClassLoader, List codeStrs) { + this.classpath = classpath; + this.parentClassLoader = parentClassLoader; + this.codeStrs = codeStrs; + this.loadClass = new HashMap>(codeStrs.size()); + } + + + /** + * 编译并且加载给定的java编码类 + * + * @throws Exception + */ + public void compileAndLoadClass() throws Exception { + String[] sourceFiles = this.uploadSrcFile(); + this.compile(sourceFiles); + this.loadClass(this.loadClass.keySet()); + } + + + public static String getClassName(String code) { + String className = StringUtil.substringBefore(code, "{"); + if (StringUtil.isBlank(className)) { + return className; + } + if (StringUtil.contains(code, " class ")) { + className = StringUtil.substringAfter(className, " class "); + if (StringUtil.contains(className, " extends ")) { + className = StringUtil.substringBefore(className, " extends ").trim(); + } + else if (StringUtil.contains(className, " implements ")) { + className = StringUtil.trim(StringUtil.substringBefore(className, " implements ")); + } + else { + className = StringUtil.trim(className); + } + } + else if (StringUtil.contains(code, " interface ")) { + className = StringUtil.substringAfter(className, " interface "); + if (StringUtil.contains(className, " extends ")) { + className = StringUtil.substringBefore(className, " extends ").trim(); + } + else { + className = StringUtil.trim(className); + } + } + else if (StringUtil.contains(code, " enum ")) { + className = StringUtil.trim(StringUtil.substringAfter(className, " enum ")); + } + else { + return StringUtil.EMPTY_STRING; + } + return className; + } + + + public static String getPackageName(String code) { + String packageName = + StringUtil.substringBefore(StringUtil.substringAfter(code, "package "), ";").trim(); + return packageName; + } + + + public static String getQualifiedName(String code) { + StringBuilder sb = new StringBuilder(); + String className = getClassName(code); + if (StringUtil.isNotBlank(className)) { + + String packageName = getPackageName(code); + if (StringUtil.isNotBlank(packageName)) { + sb.append(packageName).append("."); + } + sb.append(className); + } + return sb.toString(); + } + + + public static String getFullClassName(String code) { + String packageName = getPackageName(code); + String className = getClassName(code); + return StringUtil.isBlank(packageName) ? className : packageName + "." + className; + } + + + /** + * 加载给定className的class + * + * @param classFullNames + * @throws ClassNotFoundException + * @throws MalformedURLException + */ + private void loadClass(Set classFullNames) throws ClassNotFoundException, MalformedURLException { + synchronized (loadClass) { + // 使用outPutClassPath的URL创建个新的ClassLoader + ClassLoader classLoader = + new URLClassLoader(new URL[] { new File(outPutClassPath).toURI().toURL() }, + parentClassLoader); + for (String key : classFullNames) { + Class classz = classLoader.loadClass(key); + if (null != classz) { + loadClass.put(key, classz); + logger.info("Dyna Load Java Class File OK:----> className: " + key); + } + else { + logger.error("Dyna Load Java Class File Fail:----> className: " + key); + } + } + } + } + + + /** + * 编译给定文件绝对路径的java文件列表 + * + * @param srcFiles + * @throws Exception + */ + private void compile(String[] srcFiles) throws Exception { + String args[] = this.buildCompileJavacArgs(srcFiles); + ByteArrayOutputStream err = new ByteArrayOutputStream(); + JavaCompiler compiler = ToolProvider.getSystemJavaCompiler(); + if (compiler == null) { + throw new NullPointerException( + "ToolProvider.getSystemJavaCompiler() return null,please use JDK replace JRE!"); + } + int resultCode = compiler.run(null, null, err, args); + if (resultCode != 0) { + throw new Exception(err.toString()); + } + } + + + /** + * 将给定code的text生成java文件，并且写入硬盘 + * + * @return + * @throws Exception + */ + private String[] uploadSrcFile() throws Exception { + List srcFileAbsolutePaths = new ArrayList(codeStrs.size()); + for (String code : codeStrs) { + if (StringUtil.isNotBlank(code)) { + String packageName = getPackageName(code); + String className = getClassName(code); + if (StringUtil.isNotBlank(className)) { + File srcFile = null; + BufferedWriter bufferWriter = null; + try { + if (StringUtil.isBlank(packageName)) { + File pathFile = new File(sourcePath); + // 如果不存在就创建 + if (!pathFile.exists()) { + if (!pathFile.mkdirs()) { + throw new RuntimeException("create PathFile Error!"); + } + } + srcFile = new File(sourcePath + FILE_SP + className + ".java"); + } + else { + String srcPath = StringUtil.replace(packageName, ".", FILE_SP); + File pathFile = new File(sourcePath + FILE_SP + srcPath); + // 如果不存在就创建 + if (!pathFile.exists()) { + if (!pathFile.mkdirs()) { + throw new RuntimeException("create PathFile Error!"); + } + } + srcFile = new File(pathFile.getAbsolutePath() + FILE_SP + className + ".java"); + } + synchronized (loadClass) { + loadClass.put(getFullClassName(code), null); + } + if (null != srcFile) { + logger.warn("Dyna Create Java Source File:---->" + srcFile.getAbsolutePath()); + srcFileAbsolutePaths.add(srcFile.getAbsolutePath()); + srcFile.deleteOnExit(); + } + OutputStreamWriter outputStreamWriter = + new OutputStreamWriter(new FileOutputStream(srcFile), encoding); + bufferWriter = new BufferedWriter(outputStreamWriter); + for (String lineCode : code.split(LINE_SP)) { + bufferWriter.write(lineCode); + bufferWriter.newLine(); + } + bufferWriter.flush(); + } + finally { + if (null != bufferWriter) { + bufferWriter.close(); + } + } + } + } + } + return srcFileAbsolutePaths.toArray(new String[srcFileAbsolutePaths.size()]); + } + + + /** + * 根据给定文件列表和当前的编译参数来构建 调用javac的编译参数数组 + * + * @param srcFiles + * @return + */ + private String[] buildCompileJavacArgs(String srcFiles[]) { + ArrayList args = new ArrayList(); + if (StringUtil.isNotBlank(classpath)) { + args.add("-classpath"); + args.add(classpath); + } + if (StringUtil.isNotBlank(outPutClassPath)) { + args.add("-d"); + args.add(outPutClassPath); + } + if (StringUtil.isNotBlank(sourcePath)) { + args.add("-sourcepath"); + args.add(sourcePath); + } + if (StringUtil.isNotBlank(bootclasspath)) { + args.add("-bootclasspath"); + args.add(bootclasspath); + } + if (StringUtil.isNotBlank(extdirs)) { + args.add("-extdirs"); + args.add(extdirs); + } + if (StringUtil.isNotBlank(encoding)) { + args.add("-encoding"); + args.add(encoding); + } + if (StringUtil.isNotBlank(target)) { + args.add("-target"); + args.add(target); + } + for (int i = 0; i < srcFiles.length; i++) { + args.add(srcFiles[i]); + } + return args.toArray(new String[args.size()]); + } + + + /** + * 根据给定的classLoader获取其对应的classPath的完整字符串 路径 URLClassLoader. + */ + private static String extractClasspath(ClassLoader cl) { + StringBuffer buf = new StringBuffer(); + while (cl != null) { + if (cl instanceof URLClassLoader) { + URL urls[] = ((URLClassLoader) cl).getURLs(); + for (int i = 0; i < urls.length; i++) { + if (buf.length() > 0) { + buf.append(File.pathSeparatorChar); + } + String s = urls[i].getFile(); + try { + s = URLDecoder.decode(s, "UTF-8"); + } + catch (UnsupportedEncodingException e) { + continue; + } + File f = new File(s); + buf.append(f.getAbsolutePath()); + } + } + cl = cl.getParent(); + } + return buf.toString(); + } + + + public String getOutPutClassPath() { + return outPutClassPath; + } + + + public void setOutPutClassPath(String outPutClassPath) { + this.outPutClassPath = outPutClassPath; + } + + + public String getSourcePath() { + return sourcePath; + } + + + public void setSourcePath(String sourcePath) { + this.sourcePath = sourcePath; + } + + + public ClassLoader getParentClassLoader() { + return parentClassLoader; + } + + + public void setParentClassLoader(ClassLoader parentClassLoader) { + this.parentClassLoader = parentClassLoader; + } + + + public String getClasspath() { + return classpath; + } + + + public void setClasspath(String classpath) { + this.classpath = classpath; + } + + + public String getBootclasspath() { + return bootclasspath; + } + + + public void setBootclasspath(String bootclasspath) { + this.bootclasspath = bootclasspath; + } + + + public String getExtdirs() { + return extdirs; + } + + + public void setExtdirs(String extdirs) { + this.extdirs = extdirs; + } + + + public String getEncoding() { + return encoding; + } + + + public void setEncoding(String encoding) { + this.encoding = encoding; + } + + + public String getTarget() { + return target; + } + + + public void setTarget(String target) { + this.target = target; + } + + + public Map> getLoadClass() { + return loadClass; + } + + + public static Class compileAndLoadClass(final String className, final String javaSource) + throws Exception { + String classSimpleName = FilterAPI.simpleClassName(className); + String javaCode = new String(javaSource); + // Java类名需要替换，否则可能会产生Source变更，但是无法加载的类冲突问题 + final String newClassSimpleName = classSimpleName + System.currentTimeMillis(); + String newJavaCode = javaCode.replaceAll(classSimpleName, newClassSimpleName); + + List codes = new ArrayList(); + codes.add(newJavaCode); + // 创建DynaCode + DynaCode dc = new DynaCode(codes); + // 执行编译并且load + dc.compileAndLoadClass(); + // 获取对应的clazz + Map> map = dc.getLoadClass(); + // 反射执行结果 + Class clazz = map.get(getQualifiedName(newJavaCode)); + return clazz; + } +} \ No newline at end of file diff --git a/rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/filter/FilterClassFetchMethod.java b/rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/filter/FilterClassFetchMethod.java new file mode 100644 index 000000000..b100947c8 --- /dev/null +++ b/rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/filter/FilterClassFetchMethod.java @@ -0,0 +1,5 @@ +package com.alibaba.rocketmq.filtersrv.filter; + +public interface FilterClassFetchMethod { + public String fetch(final String topic, final String consumerGroup, final String className); +} diff --git a/rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/filter/FilterClassInfo.java b/rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/filter/FilterClassInfo.java new file mode 100644 index 000000000..65c2fd951 --- /dev/null +++ b/rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/filter/FilterClassInfo.java @@ -0,0 +1,40 @@ +package com.alibaba.rocketmq.filtersrv.filter; + +import com.alibaba.rocketmq.common.filter.MessageFilter; + + +public class FilterClassInfo { + private String className; + private int classCRC; + private MessageFilter messageFilter; + + + public int getClassCRC() { + return classCRC; + } + + + public void setClassCRC(int classCRC) { + this.classCRC = classCRC; + } + + + public MessageFilter getMessageFilter() { + return messageFilter; + } + + + public void setMessageFilter(MessageFilter messageFilter) { + this.messageFilter = messageFilter; + } + + + public String getClassName() { + return className; + } + + + public void setClassName(String className) { + this.className = className; + } +} diff --git a/rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/filter/FilterClassLoader.java b/rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/filter/FilterClassLoader.java new file mode 100644 index 000000000..8b76f07a5 --- /dev/null +++ b/rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/filter/FilterClassLoader.java @@ -0,0 +1,7 @@ +package com.alibaba.rocketmq.filtersrv.filter; + +public class FilterClassLoader extends ClassLoader { + public final Class createNewClass(String name, byte[] b, int off, int len) throws ClassFormatError { + return this.defineClass(name, b, off, len); + } +} diff --git a/rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/filter/FilterClassManager.java b/rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/filter/FilterClassManager.java new file mode 100644 index 000000000..e8843c2d3 --- /dev/null +++ b/rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/filter/FilterClassManager.java @@ -0,0 +1,168 @@ +package com.alibaba.rocketmq.filtersrv.filter; + +import java.util.Iterator; +import java.util.Map.Entry; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.ThreadFactoryImpl; +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.common.filter.MessageFilter; +import com.alibaba.rocketmq.filtersrv.FiltersrvController; + + +public class FilterClassManager { + private static final Logger log = LoggerFactory.getLogger(LoggerName.FiltersrvLoggerName); + + private ConcurrentHashMap filterClassTable = + new ConcurrentHashMap(128); + + // 只为编译加锁使用 + private final Object compileLock = new Object(); + + private final FiltersrvController filtersrvController; + + // 定时线程 + private final ScheduledExecutorService scheduledExecutorService = Executors + .newSingleThreadScheduledExecutor(new ThreadFactoryImpl("FSGetClassScheduledThread")); + + private FilterClassFetchMethod filterClassFetchMethod; + + + public FilterClassManager(FiltersrvController filtersrvController) { + this.filtersrvController = filtersrvController; + this.filterClassFetchMethod = + new HttpFilterClassFetchMethod(this.filtersrvController.getFiltersrvConfig() + .getFilterClassRepertoryUrl()); + } + + + public void start() { + if (!this.filtersrvController.getFiltersrvConfig().isClientUploadFilterClassEnable()) { + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + + @Override + public void run() { + fetchClassFromRemoteHost(); + } + }, 1, 1, TimeUnit.MINUTES); + } + } + + + public void shutdown() { + this.scheduledExecutorService.shutdown(); + } + + + private void fetchClassFromRemoteHost() { + Iterator> it = this.filterClassTable.entrySet().iterator(); + while (it.hasNext()) { + try { + Entry next = it.next(); + FilterClassInfo filterClassInfo = next.getValue(); + String[] topicAndGroup = next.getKey().split("@"); + String responseStr = + this.filterClassFetchMethod.fetch(topicAndGroup[0], topicAndGroup[1], + filterClassInfo.getClassName()); + byte[] filterSourceBinary = responseStr.getBytes("UTF-8"); + int classCRC = UtilAll.crc32(responseStr.getBytes("UTF-8")); + if (classCRC != filterClassInfo.getClassCRC()) { + String javaSource = new String(filterSourceBinary, MixAll.DEFAULT_CHARSET); + Class newClass = + DynaCode.compileAndLoadClass(filterClassInfo.getClassName(), javaSource); + Object newInstance = newClass.newInstance(); + filterClassInfo.setMessageFilter((MessageFilter) newInstance); + filterClassInfo.setClassCRC(classCRC); + + log.info("fetch Remote class File OK, {} {}", next.getKey(), + filterClassInfo.getClassName()); + } + } + catch (Exception e) { + log.error("fetchClassFromRemoteHost Exception", e); + } + } + } + + + private static String buildKey(final String consumerGroup, final String topic) { + return topic + "@" + consumerGroup; + } + + + public boolean registerFilterClass(final String consumerGroup, final String topic, + final String className, final int classCRC, final byte[] filterSourceBinary) { + final String key = buildKey(consumerGroup, topic); + + // 先检查是否存在，是否CRC相同 + boolean registerNew = false; + FilterClassInfo filterClassInfoPrev = this.filterClassTable.get(key); + if (null == filterClassInfoPrev) { + registerNew = true; + } + else { + if (this.filtersrvController.getFiltersrvConfig().isClientUploadFilterClassEnable()) { + if (filterClassInfoPrev.getClassCRC() != classCRC && classCRC != 0) { + registerNew = true; + } + } + } + + // 注册新的Class + if (registerNew) { + synchronized (this.compileLock) { + filterClassInfoPrev = this.filterClassTable.get(key); + if (null != filterClassInfoPrev && filterClassInfoPrev.getClassCRC() == classCRC) { + return true; + } + + try { + + FilterClassInfo filterClassInfoNew = new FilterClassInfo(); + filterClassInfoNew.setClassName(className); + filterClassInfoNew.setClassCRC(0); + filterClassInfoNew.setMessageFilter(null); + + if (this.filtersrvController.getFiltersrvConfig().isClientUploadFilterClassEnable()) { + String javaSource = new String(filterSourceBinary, MixAll.DEFAULT_CHARSET); + Class newClass = DynaCode.compileAndLoadClass(className, javaSource); + Object newInstance = newClass.newInstance(); + filterClassInfoNew.setMessageFilter((MessageFilter) newInstance); + filterClassInfoNew.setClassCRC(classCRC); + } + + this.filterClassTable.put(key, filterClassInfoNew); + } + catch (Throwable e) { + log.error("compileAndLoadClass Exception", e); + return false; + } + } + } + + return true; + } + + + public FilterClassInfo findFilterClass(final String consumerGroup, final String topic) { + return this.filterClassTable.get(buildKey(consumerGroup, topic)); + } + + + public FilterClassFetchMethod getFilterClassFetchMethod() { + return filterClassFetchMethod; + } + + + public void setFilterClassFetchMethod(FilterClassFetchMethod filterClassFetchMethod) { + this.filterClassFetchMethod = filterClassFetchMethod; + } +} diff --git a/rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/filter/HttpFilterClassFetchMethod.java b/rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/filter/HttpFilterClassFetchMethod.java new file mode 100644 index 000000000..b4f484506 --- /dev/null +++ b/rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/filter/HttpFilterClassFetchMethod.java @@ -0,0 +1,38 @@ +package com.alibaba.rocketmq.filtersrv.filter; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.common.utils.HttpTinyClient; +import com.alibaba.rocketmq.common.utils.HttpTinyClient.HttpResult; + + +public class HttpFilterClassFetchMethod implements FilterClassFetchMethod { + private static final Logger log = LoggerFactory.getLogger(LoggerName.FiltersrvLoggerName); + private final String url; + + + public HttpFilterClassFetchMethod(String url) { + this.url = url; + } + + + @Override + public String fetch(String topic, String consumerGroup, String className) { + String thisUrl = String.format("%s/%s.java", this.url, className); + + try { + HttpResult result = HttpTinyClient.httpGet(thisUrl, null, null, "UTF-8", 5000); + if (200 == result.code) { + return result.content; + } + } + catch (Exception e) { + log.error( + String.format("call <%s> exception, Topic: %s Group: %s", thisUrl, topic, consumerGroup), e); + } + + return null; + } +} diff --git a/rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/processor/DefaultRequestProcessor.java b/rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/processor/DefaultRequestProcessor.java new file mode 100644 index 000000000..585a0cf8e --- /dev/null +++ b/rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/processor/DefaultRequestProcessor.java @@ -0,0 +1,379 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.filtersrv.processor; + +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelFutureListener; +import io.netty.channel.ChannelHandlerContext; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.rocketmq.client.consumer.DefaultMQPullConsumer; +import com.alibaba.rocketmq.client.consumer.PullCallback; +import com.alibaba.rocketmq.client.consumer.PullResult; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.common.message.MessageDecoder; +import com.alibaba.rocketmq.common.message.MessageExt; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.common.protocol.RequestCode; +import com.alibaba.rocketmq.common.protocol.ResponseCode; +import com.alibaba.rocketmq.common.protocol.header.PullMessageRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.PullMessageResponseHeader; +import com.alibaba.rocketmq.common.protocol.header.filtersrv.RegisterMessageFilterClassRequestHeader; +import com.alibaba.rocketmq.common.sysflag.MessageSysFlag; +import com.alibaba.rocketmq.filtersrv.FiltersrvController; +import com.alibaba.rocketmq.filtersrv.filter.FilterClassInfo; +import com.alibaba.rocketmq.remoting.common.RemotingHelper; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; +import com.alibaba.rocketmq.remoting.netty.NettyRequestProcessor; +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; +import com.alibaba.rocketmq.store.CommitLog; + + +/** + * Filter Server网络请求处理 + * + * @author shijia.wxr + * @since 2014-4-20 + */ +public class DefaultRequestProcessor implements NettyRequestProcessor { + private static final Logger log = LoggerFactory.getLogger(LoggerName.FiltersrvLoggerName); + + private final FiltersrvController filtersrvController; + + + public DefaultRequestProcessor(FiltersrvController filtersrvController) { + this.filtersrvController = filtersrvController; + } + + + @Override + public RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand request) + throws Exception { + if (log.isDebugEnabled()) { + log.debug("receive request, {} {} {}",// + request.getCode(), // + RemotingHelper.parseChannelRemoteAddr(ctx.channel()), // + request); + } + + switch (request.getCode()) { + case RequestCode.REGISTER_MESSAGE_FILTER_CLASS: + return registerMessageFilterClass(ctx, request); + case RequestCode.PULL_MESSAGE: + return pullMessageForward(ctx, request); + } + + return null; + } + + + private ByteBuffer messageToByteBuffer(final MessageExt msg) throws IOException { + int sysFlag = MessageSysFlag.clearCompressedFlag(msg.getSysFlag()); + if (msg.getBody() != null) { + if (msg.getBody().length >= this.filtersrvController.getFiltersrvConfig() + .getCompressMsgBodyOverHowmuch()) { + byte[] data = + UtilAll.compress(msg.getBody(), this.filtersrvController.getFiltersrvConfig() + .getZipCompressLevel()); + if (data != null) { + msg.setBody(data); + sysFlag |= MessageSysFlag.CompressedFlag; + } + } + } + + final int bodyLength = msg.getBody() != null ? msg.getBody().length : 0; + byte[] topicData = msg.getTopic().getBytes(MixAll.DEFAULT_CHARSET); + final int topicLength = topicData.length; + String properties = MessageDecoder.messageProperties2String(msg.getProperties()); + byte[] propertiesData = properties.getBytes(MixAll.DEFAULT_CHARSET); + final int propertiesLength = propertiesData.length; + final int msgLen = 4 // 1 TOTALSIZE + + 4 // 2 MAGICCODE + + 4 // 3 BODYCRC + + 4 // 4 QUEUEID + + 4 // 5 FLAG + + 8 // 6 QUEUEOFFSET + + 8 // 7 PHYSICALOFFSET + + 4 // 8 SYSFLAG + + 8 // 9 BORNTIMESTAMP + + 8 // 10 BORNHOST + + 8 // 11 STORETIMESTAMP + + 8 // 12 STOREHOSTADDRESS + + 4 // 13 RECONSUMETIMES + + 8 // 14 Prepared Transaction Offset + + 4 + bodyLength // 14 BODY + + 1 + topicLength // 15 TOPIC + + 2 + propertiesLength // 16 propertiesLength + + 0; + + ByteBuffer msgStoreItemMemory = ByteBuffer.allocate(msgLen); + + final MessageExt msgInner = msg; + + // 1 TOTALSIZE + msgStoreItemMemory.putInt(msgLen); + // 2 MAGICCODE + msgStoreItemMemory.putInt(CommitLog.MessageMagicCode); + // 3 BODYCRC + msgStoreItemMemory.putInt(UtilAll.crc32(msgInner.getBody())); + // 4 QUEUEID + msgStoreItemMemory.putInt(msgInner.getQueueId()); + // 5 FLAG + msgStoreItemMemory.putInt(msgInner.getFlag()); + // 6 QUEUEOFFSET + msgStoreItemMemory.putLong(msgInner.getQueueOffset()); + // 7 PHYSICALOFFSET + msgStoreItemMemory.putLong(msgInner.getCommitLogOffset()); + // 8 SYSFLAG + msgStoreItemMemory.putInt(sysFlag); + // 9 BORNTIMESTAMP + msgStoreItemMemory.putLong(msgInner.getBornTimestamp()); + // 10 BORNHOST + msgStoreItemMemory.put(msgInner.getBornHostBytes()); + // 11 STORETIMESTAMP + msgStoreItemMemory.putLong(msgInner.getStoreTimestamp()); + // 12 STOREHOSTADDRESS + msgStoreItemMemory.put(msgInner.getStoreHostBytes()); + // 13 RECONSUMETIMES + msgStoreItemMemory.putInt(msgInner.getReconsumeTimes()); + // 14 Prepared Transaction Offset + msgStoreItemMemory.putLong(msgInner.getPreparedTransactionOffset()); + // 15 BODY + msgStoreItemMemory.putInt(bodyLength); + if (bodyLength > 0) + msgStoreItemMemory.put(msgInner.getBody()); + // 16 TOPIC + msgStoreItemMemory.put((byte) topicLength); + msgStoreItemMemory.put(topicData); + // 17 PROPERTIES + msgStoreItemMemory.putShort((short) propertiesLength); + if (propertiesLength > 0) + msgStoreItemMemory.put(propertiesData); + + return msgStoreItemMemory; + } + + + private void returnResponse(final String group, final String topic, ChannelHandlerContext ctx, + final RemotingCommand response, final List msgList) { + if (null != msgList) { + ByteBuffer[] msgBufferList = new ByteBuffer[msgList.size()]; + int bodyTotalSize = 0; + for (int i = 0; i < msgList.size(); i++) { + try { + msgBufferList[i] = messageToByteBuffer(msgList.get(i)); + bodyTotalSize += msgBufferList[i].capacity(); + } + catch (Exception e) { + log.error("messageToByteBuffer UnsupportedEncodingException", e); + } + } + + ByteBuffer body = ByteBuffer.allocate(bodyTotalSize); + for (ByteBuffer bb : msgBufferList) { + bb.flip(); + body.put(bb); + } + + response.setBody(body.array()); + + // 统计 + this.filtersrvController.getFilterServerStatsManager().incGroupGetNums(group, topic, + msgList.size()); + + this.filtersrvController.getFilterServerStatsManager().incGroupGetSize(group, topic, + bodyTotalSize); + } + + try { + ctx.writeAndFlush(response).addListener(new ChannelFutureListener() { + @Override + public void operationComplete(ChannelFuture future) throws Exception { + if (!future.isSuccess()) { + log.error("FilterServer response to " + future.channel().remoteAddress() + " failed", + future.cause()); + log.error(response.toString()); + } + } + }); + } + catch (Throwable e) { + log.error("FilterServer process request over, but response failed", e); + log.error(response.toString()); + } + } + + + private RemotingCommand pullMessageForward(final ChannelHandlerContext ctx, final RemotingCommand request) + throws Exception { + final RemotingCommand response = + RemotingCommand.createResponseCommand(PullMessageResponseHeader.class); + final PullMessageResponseHeader responseHeader = + (PullMessageResponseHeader) response.readCustomHeader(); + final PullMessageRequestHeader requestHeader = + (PullMessageRequestHeader) request.decodeCommandCustomHeader(PullMessageRequestHeader.class); + + // 由于异步返回，所以必须要设置 + response.setOpaque(request.getOpaque()); + + DefaultMQPullConsumer pullConsumer = this.filtersrvController.getDefaultMQPullConsumer(); + final FilterClassInfo findFilterClass = + this.filtersrvController.getFilterClassManager().findFilterClass( + requestHeader.getConsumerGroup(), requestHeader.getTopic()); + if (null == findFilterClass) { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("Find Filter class failed, not registered"); + return response; + } + + if (null == findFilterClass.getMessageFilter()) { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("Find Filter class failed, registered but no class"); + return response; + } + + responseHeader.setSuggestWhichBrokerId(MixAll.MASTER_ID); + + // 构造从Broker拉消息的参数 + MessageQueue mq = new MessageQueue(); + mq.setTopic(requestHeader.getTopic()); + mq.setQueueId(requestHeader.getQueueId()); + mq.setBrokerName(this.filtersrvController.getBrokerName()); + long offset = requestHeader.getQueueOffset(); + int maxNums = requestHeader.getMaxMsgNums(); + + final PullCallback pullCallback = new PullCallback() { + + @Override + public void onSuccess(PullResult pullResult) { + responseHeader.setMaxOffset(pullResult.getMaxOffset()); + responseHeader.setMinOffset(pullResult.getMinOffset()); + responseHeader.setNextBeginOffset(pullResult.getNextBeginOffset()); + response.setRemark(null); + + switch (pullResult.getPullStatus()) { + case FOUND: + response.setCode(ResponseCode.SUCCESS); + + List msgListOK = new ArrayList(); + try { + for (MessageExt msg : pullResult.getMsgFoundList()) { + boolean match = findFilterClass.getMessageFilter().match(msg); + if (match) { + msgListOK.add(msg); + } + } + + // 有消息返回 + if (!msgListOK.isEmpty()) { + returnResponse(requestHeader.getConsumerGroup(), requestHeader.getTopic(), ctx, + response, msgListOK); + return; + } + // 全部都被过滤掉了 + else { + response.setCode(ResponseCode.PULL_RETRY_IMMEDIATELY); + } + } + // 只要抛异常，就终止过滤，并返回客户端异常 + catch (Throwable e) { + final String error = + String.format("do Message Filter Exception, ConsumerGroup: %s Topic: %s ", + requestHeader.getConsumerGroup(), requestHeader.getTopic()); + log.error(error, e); + + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark(error + RemotingHelper.exceptionSimpleDesc(e)); + returnResponse(requestHeader.getConsumerGroup(), requestHeader.getTopic(), ctx, + response, null); + return; + } + + break; + case NO_MATCHED_MSG: + response.setCode(ResponseCode.PULL_RETRY_IMMEDIATELY); + break; + case NO_NEW_MSG: + response.setCode(ResponseCode.PULL_NOT_FOUND); + break; + case OFFSET_ILLEGAL: + response.setCode(ResponseCode.PULL_OFFSET_MOVED); + break; + default: + break; + } + + returnResponse(requestHeader.getConsumerGroup(), requestHeader.getTopic(), ctx, response, + null); + } + + + @Override + public void onException(Throwable e) { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark("Pull Callback Exception, " + RemotingHelper.exceptionSimpleDesc(e)); + returnResponse(requestHeader.getConsumerGroup(), requestHeader.getTopic(), ctx, response, + null); + return; + } + }; + + pullConsumer.pullBlockIfNotFound(mq, null, offset, maxNums, pullCallback); + + return null; + } + + + private RemotingCommand registerMessageFilterClass(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + final RegisterMessageFilterClassRequestHeader requestHeader = + (RegisterMessageFilterClassRequestHeader) request + .decodeCommandCustomHeader(RegisterMessageFilterClassRequestHeader.class); + + try { + boolean ok = + this.filtersrvController.getFilterClassManager().registerFilterClass( + requestHeader.getConsumerGroup(),// + requestHeader.getTopic(),// + requestHeader.getClassName(),// + requestHeader.getClassCRC(), // + request.getBody());// Body传输的是Java Source，必须UTF-8编码 + if (!ok) { + throw new Exception("registerFilterClass error"); + } + } + catch (Exception e) { + response.setCode(ResponseCode.SYSTEM_ERROR); + response.setRemark(RemotingHelper.exceptionSimpleDesc(e)); + return response; + } + + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } +} diff --git a/rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/stats/FilterServerStatsManager.java b/rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/stats/FilterServerStatsManager.java new file mode 100644 index 000000000..6e9a92dfe --- /dev/null +++ b/rocketmq-filtersrv/src/main/java/com/alibaba/rocketmq/filtersrv/stats/FilterServerStatsManager.java @@ -0,0 +1,49 @@ +package com.alibaba.rocketmq.filtersrv.stats; + +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.rocketmq.common.ThreadFactoryImpl; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.common.stats.StatsItemSet; + + +public class FilterServerStatsManager { + private static final Logger log = LoggerFactory.getLogger(LoggerName.FiltersrvLoggerName); + private final ScheduledExecutorService scheduledExecutorService = Executors + .newSingleThreadScheduledExecutor(new ThreadFactoryImpl("FSStatsThread")); + + // ConsumerGroup Get Nums + private final StatsItemSet groupGetNums = new StatsItemSet("GROUP_GET_NUMS", + this.scheduledExecutorService, log); + + // ConsumerGroup Get Size + private final StatsItemSet groupGetSize = new StatsItemSet("GROUP_GET_SIZE", + this.scheduledExecutorService, log); + + + public FilterServerStatsManager() { + } + + + public void start() { + } + + + public void shutdown() { + this.scheduledExecutorService.shutdown(); + } + + + public void incGroupGetNums(final String group, final String topic, final int incValue) { + this.groupGetNums.addValue(topic + "@" + group, incValue, 1); + } + + + public void incGroupGetSize(final String group, final String topic, final int incValue) { + this.groupGetSize.addValue(topic + "@" + group, incValue, 1); + } +} diff --git a/rocketmq-namesrv/pom.xml b/rocketmq-namesrv/pom.xml new file mode 100644 index 000000000..d7a13d0a5 --- /dev/null +++ b/rocketmq-namesrv/pom.xml @@ -0,0 +1,33 @@ + + + com.alibaba.rocketmq + rocketmq-all + 3.2.2 + + + 4.0.0 + jar + rocketmq-namesrv + rocketmq-namesrv ${project.version} + + + + junit + junit + test + + + ${project.groupId} + rocketmq-srvutil + + + ch.qos.logback + logback-classic + + + ch.qos.logback + logback-core + + + diff --git a/rocketmq-namesrv/src/main/java/com/alibaba/rocketmq/namesrv/NamesrvController.java b/rocketmq-namesrv/src/main/java/com/alibaba/rocketmq/namesrv/NamesrvController.java new file mode 100644 index 000000000..90dfedfc2 --- /dev/null +++ b/rocketmq-namesrv/src/main/java/com/alibaba/rocketmq/namesrv/NamesrvController.java @@ -0,0 +1,166 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.namesrv; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.rocketmq.common.ThreadFactoryImpl; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.common.namesrv.NamesrvConfig; +import com.alibaba.rocketmq.namesrv.kvconfig.KVConfigManager; +import com.alibaba.rocketmq.namesrv.processor.DefaultRequestProcessor; +import com.alibaba.rocketmq.namesrv.routeinfo.BrokerHousekeepingService; +import com.alibaba.rocketmq.namesrv.routeinfo.RouteInfoManager; +import com.alibaba.rocketmq.remoting.RemotingServer; +import com.alibaba.rocketmq.remoting.netty.NettyRemotingServer; +import com.alibaba.rocketmq.remoting.netty.NettyServerConfig; + + +/** + * Name Server服务控制 + * + * @author shijia.wxr + * @since 2013-7-5 + */ +public class NamesrvController { + private static final Logger log = LoggerFactory.getLogger(LoggerName.NamesrvLoggerName); + // Name Server配置 + private final NamesrvConfig namesrvConfig; + // 通信层配置 + private final NettyServerConfig nettyServerConfig; + // 服务端通信层对象 + private RemotingServer remotingServer; + // 接收Broker连接事件 + private BrokerHousekeepingService brokerHousekeepingService; + // 服务端网络请求处理线程池 + private ExecutorService remotingExecutor; + + // 定时线程 + private final ScheduledExecutorService scheduledExecutorService = Executors + .newSingleThreadScheduledExecutor(new ThreadFactoryImpl("NSScheduledThread")); + + /** + * 核心数据结构 + */ + private final KVConfigManager kvConfigManager; + private final RouteInfoManager routeInfoManager; + + + public NamesrvController(NamesrvConfig namesrvConfig, NettyServerConfig nettyServerConfig) { + this.namesrvConfig = namesrvConfig; + this.nettyServerConfig = nettyServerConfig; + this.kvConfigManager = new KVConfigManager(this); + this.routeInfoManager = new RouteInfoManager(); + this.brokerHousekeepingService = new BrokerHousekeepingService(this); + } + + + public boolean initialize() { + // 加载KV配置 + this.kvConfigManager.load(); + + // 初始化通信层 + this.remotingServer = new NettyRemotingServer(this.nettyServerConfig, this.brokerHousekeepingService); + + // 初始化线程池 + this.remotingExecutor = + Executors.newFixedThreadPool(nettyServerConfig.getServerWorkerThreads(), + new ThreadFactoryImpl("RemotingExecutorThread_")); + + this.registerProcessor(); + + // 增加定时任务 + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + + @Override + public void run() { + NamesrvController.this.routeInfoManager.scanNotActiveBroker(); + } + }, 5, 10, TimeUnit.SECONDS); + + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + + @Override + public void run() { + NamesrvController.this.kvConfigManager.printAllPeriodically(); + } + }, 1, 10, TimeUnit.MINUTES); + + // this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + // + // @Override + // public void run() { + // NamesrvController.this.routeInfoManager.printAllPeriodically(); + // } + // }, 1, 5, TimeUnit.MINUTES); + + return true; + } + + + private void registerProcessor() { + this.remotingServer + .registerDefaultProcessor(new DefaultRequestProcessor(this), this.remotingExecutor); + } + + + public void start() throws Exception { + this.remotingServer.start(); + } + + + public void shutdown() { + this.remotingServer.shutdown(); + this.remotingExecutor.shutdown(); + this.scheduledExecutorService.shutdown(); + } + + + public NamesrvConfig getNamesrvConfig() { + return namesrvConfig; + } + + + public NettyServerConfig getNettyServerConfig() { + return nettyServerConfig; + } + + + public KVConfigManager getKvConfigManager() { + return kvConfigManager; + } + + + public RouteInfoManager getRouteInfoManager() { + return routeInfoManager; + } + + + public RemotingServer getRemotingServer() { + return remotingServer; + } + + + public void setRemotingServer(RemotingServer remotingServer) { + this.remotingServer = remotingServer; + } +} diff --git a/rocketmq-namesrv/src/main/java/com/alibaba/rocketmq/namesrv/NamesrvStartup.java b/rocketmq-namesrv/src/main/java/com/alibaba/rocketmq/namesrv/NamesrvStartup.java new file mode 100644 index 000000000..ce489cc07 --- /dev/null +++ b/rocketmq-namesrv/src/main/java/com/alibaba/rocketmq/namesrv/NamesrvStartup.java @@ -0,0 +1,185 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.namesrv; + +import java.io.BufferedInputStream; +import java.io.FileInputStream; +import java.io.InputStream; +import java.util.Properties; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.PosixParser; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import ch.qos.logback.classic.LoggerContext; +import ch.qos.logback.classic.joran.JoranConfigurator; + +import com.alibaba.rocketmq.common.MQVersion; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.common.namesrv.NamesrvConfig; +import com.alibaba.rocketmq.remoting.netty.NettyServerConfig; +import com.alibaba.rocketmq.remoting.netty.NettySystemConfig; +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; +import com.alibaba.rocketmq.srvutil.ServerUtil; + + +/** + * Name server 启动入口 + * + * @author shijia.wxr + * @since 2013-7-5 + */ +public class NamesrvStartup { + public static Properties properties = null; + public static CommandLine commandLine = null; + + + public static Options buildCommandlineOptions(final Options options) { + Option opt = new Option("c", "configFile", true, "Name server config properties file"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("p", "printConfigItem", false, "Print all config item"); + opt.setRequired(false); + options.addOption(opt); + + return options; + } + + + public static void main(String[] args) { + main0(args); + } + + + public static NamesrvController main0(String[] args) { + System.setProperty(RemotingCommand.RemotingVersionKey, Integer.toString(MQVersion.CurrentVersion)); + + // Socket发送缓冲区大小 + if (null == System.getProperty(NettySystemConfig.SystemPropertySocketSndbufSize)) { + NettySystemConfig.SocketSndbufSize = 2048; + } + + // Socket接收缓冲区大小 + if (null == System.getProperty(NettySystemConfig.SystemPropertySocketRcvbufSize)) { + NettySystemConfig.SocketRcvbufSize = 1024; + } + + try { + // 解析命令行 + Options options = ServerUtil.buildCommandlineOptions(new Options()); + commandLine = + ServerUtil.parseCmdLine("mqnamesrv", args, buildCommandlineOptions(options), + new PosixParser()); + if (null == commandLine) { + System.exit(-1); + return null; + } + + // 初始化配置文件 + final NamesrvConfig namesrvConfig = new NamesrvConfig(); + final NettyServerConfig nettyServerConfig = new NettyServerConfig(); + nettyServerConfig.setListenPort(9876); + if (commandLine.hasOption('c')) { + String file = commandLine.getOptionValue('c'); + if (file != null) { + InputStream in = new BufferedInputStream(new FileInputStream(file)); + properties = new Properties(); + properties.load(in); + MixAll.properties2Object(properties, namesrvConfig); + MixAll.properties2Object(properties, nettyServerConfig); + System.out.println("load config properties file OK, " + file); + in.close(); + } + } + + // 打印默认配置 + if (commandLine.hasOption('p')) { + MixAll.printObjectProperties(null, namesrvConfig); + MixAll.printObjectProperties(null, nettyServerConfig); + System.exit(0); + } + + MixAll.properties2Object(ServerUtil.commandLine2Properties(commandLine), namesrvConfig); + + if (null == namesrvConfig.getRocketmqHome()) { + System.out.println("Please set the " + MixAll.ROCKETMQ_HOME_ENV + + " variable in your environment to match the location of the RocketMQ installation"); + System.exit(-2); + } + + // 初始化Logback + LoggerContext lc = (LoggerContext) LoggerFactory.getILoggerFactory(); + JoranConfigurator configurator = new JoranConfigurator(); + configurator.setContext(lc); + lc.reset(); + configurator.doConfigure(namesrvConfig.getRocketmqHome() + "/conf/logback_namesrv.xml"); + final Logger log = LoggerFactory.getLogger(LoggerName.NamesrvLoggerName); + + // 打印服务器配置参数 + MixAll.printObjectProperties(log, namesrvConfig); + MixAll.printObjectProperties(log, nettyServerConfig); + + // 初始化服务控制对象 + final NamesrvController controller = new NamesrvController(namesrvConfig, nettyServerConfig); + boolean initResult = controller.initialize(); + if (!initResult) { + controller.shutdown(); + System.exit(-3); + } + + Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() { + private volatile boolean hasShutdown = false; + private AtomicInteger shutdownTimes = new AtomicInteger(0); + + + @Override + public void run() { + synchronized (this) { + log.info("shutdown hook was invoked, " + this.shutdownTimes.incrementAndGet()); + if (!this.hasShutdown) { + this.hasShutdown = true; + long begineTime = System.currentTimeMillis(); + controller.shutdown(); + long consumingTimeTotal = System.currentTimeMillis() - begineTime; + log.info("shutdown hook over, consuming time total(ms): " + consumingTimeTotal); + } + } + } + }, "ShutdownHook")); + + // 启动服务 + controller.start(); + + String tip = "The Name Server boot success."; + log.info(tip); + System.out.println(tip); + + return controller; + } + catch (Throwable e) { + e.printStackTrace(); + System.exit(-1); + } + + return null; + } +} diff --git a/rocketmq-namesrv/src/main/java/com/alibaba/rocketmq/namesrv/kvconfig/KVConfigManager.java b/rocketmq-namesrv/src/main/java/com/alibaba/rocketmq/namesrv/kvconfig/KVConfigManager.java new file mode 100644 index 000000000..2aa382428 --- /dev/null +++ b/rocketmq-namesrv/src/main/java/com/alibaba/rocketmq/namesrv/kvconfig/KVConfigManager.java @@ -0,0 +1,283 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.namesrv.kvconfig; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.Map.Entry; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.common.protocol.body.KVTable; +import com.alibaba.rocketmq.namesrv.NamesrvController; + + +/** + * KV配置管理 + * + * @author shijia.wxr + * @since 2013-7-1 + */ +public class KVConfigManager { + private static final Logger log = LoggerFactory.getLogger(LoggerName.NamesrvLoggerName); + + private final NamesrvController namesrvController; + + private final ReadWriteLock lock = new ReentrantReadWriteLock(); + private final HashMap> configTable = + new HashMap>(); + + + public KVConfigManager(NamesrvController namesrvController) { + this.namesrvController = namesrvController; + } + + + public void load() { + String content = MixAll.file2String(this.namesrvController.getNamesrvConfig().getKvConfigPath()); + if (content != null) { + KVConfigSerializeWrapper kvConfigSerializeWrapper = + KVConfigSerializeWrapper.fromJson(content, KVConfigSerializeWrapper.class); + if (null != kvConfigSerializeWrapper) { + this.configTable.putAll(kvConfigSerializeWrapper.getConfigTable()); + log.info("load KV config table OK"); + } + } + } + + + public void putKVConfig(final String namespace, final String key, final String value) { + try { + this.lock.writeLock().lockInterruptibly(); + try { + HashMap kvTable = this.configTable.get(namespace); + if (null == kvTable) { + kvTable = new HashMap(); + this.configTable.put(namespace, kvTable); + log.info("putKVConfig create new Namespace {}", namespace); + } + + final String prev = kvTable.put(key, value); + if (null != prev) { + log.info("putKVConfig update config item, Namespace: {} Key: {} Value: {}", // + namespace, key, value); + } + else { + log.info("putKVConfig create new config item, Namespace: {} Key: {} Value: {}", // + namespace, key, value); + } + } + finally { + this.lock.writeLock().unlock(); + } + } + catch (InterruptedException e) { + log.error("putKVConfig InterruptedException", e); + } + + this.persist(); + } + + + public void deleteKVConfig(final String namespace, final String key) { + try { + this.lock.writeLock().lockInterruptibly(); + try { + HashMap kvTable = this.configTable.get(namespace); + if (null != kvTable) { + String value = kvTable.remove(key); + log.info("deleteKVConfig delete a config item, Namespace: {} Key: {} Value: {}", // + namespace, key, value); + } + } + finally { + this.lock.writeLock().unlock(); + } + } + catch (InterruptedException e) { + log.error("deleteKVConfig InterruptedException", e); + } + + this.persist(); + } + + + public byte[] getKVListByNamespace(final String namespace) { + try { + this.lock.readLock().lockInterruptibly(); + try { + HashMap kvTable = this.configTable.get(namespace); + if (null != kvTable) { + KVTable table = new KVTable(); + table.setTable(kvTable); + return table.encode(); + } + } + finally { + this.lock.readLock().unlock(); + } + } + catch (InterruptedException e) { + log.error("getKVListByNamespace InterruptedException", e); + } + + return null; + } + + + public String getKVConfig(final String namespace, final String key) { + try { + this.lock.readLock().lockInterruptibly(); + try { + HashMap kvTable = this.configTable.get(namespace); + if (null != kvTable) { + return kvTable.get(key); + } + } + finally { + this.lock.readLock().unlock(); + } + } + catch (InterruptedException e) { + log.error("getKVConfig InterruptedException", e); + } + + return null; + } + + + public String getKVConfigByValue(final String namespace, final String value) { + try { + this.lock.readLock().lockInterruptibly(); + try { + HashMap kvTable = this.configTable.get(namespace); + if (null != kvTable) { + StringBuilder sb = new StringBuilder(); + String splitor = ""; + for (Map.Entry entry : kvTable.entrySet()) { + if (value.equals(entry.getValue())) { + sb.append(splitor).append(entry.getKey()); + splitor = ";"; + } + } + return sb.toString(); + } + } + finally { + this.lock.readLock().unlock(); + } + } + catch (InterruptedException e) { + log.error("getIpsByProjectGroup InterruptedException", e); + } + + return null; + } + + + public void deleteKVConfigByValue(final String namespace, final String value) { + try { + this.lock.writeLock().lockInterruptibly(); + try { + HashMap kvTable = this.configTable.get(namespace); + if (null != kvTable) { + HashMap cloneKvTable = new HashMap(kvTable); + for (Map.Entry entry : cloneKvTable.entrySet()) { + if (value.equals(entry.getValue())) { + kvTable.remove(entry.getKey()); + } + } + log.info("deleteIpsByProjectGroup delete a config item, Namespace: {} Key: {} Value: {}", // + namespace, value); + } + } + finally { + this.lock.writeLock().unlock(); + } + } + catch (InterruptedException e) { + log.error("deleteIpsByProjectGroup InterruptedException", e); + } + + this.persist(); + } + + + public void persist() { + try { + this.lock.readLock().lockInterruptibly(); + try { + KVConfigSerializeWrapper kvConfigSerializeWrapper = new KVConfigSerializeWrapper(); + kvConfigSerializeWrapper.setConfigTable(this.configTable); + + String content = kvConfigSerializeWrapper.toJson(); + + if (null != content) { + MixAll.string2File(content, this.namesrvController.getNamesrvConfig().getKvConfigPath()); + } + } + catch (IOException e) { + log.error("persist kvconfig Exception, " + + this.namesrvController.getNamesrvConfig().getKvConfigPath(), e); + } + finally { + this.lock.readLock().unlock(); + } + } + catch (InterruptedException e) { + log.error("persist InterruptedException", e); + } + + } + + + public void printAllPeriodically() { + try { + this.lock.readLock().lockInterruptibly(); + try { + log.info("--------------------------------------------------------"); + + { + log.info("configTable SIZE: {}", this.configTable.size()); + Iterator>> it = + this.configTable.entrySet().iterator(); + while (it.hasNext()) { + Entry> next = it.next(); + Iterator> itSub = next.getValue().entrySet().iterator(); + while (itSub.hasNext()) { + Entry nextSub = itSub.next(); + log.info("configTable NS: {} Key: {} Value: {}", next.getKey(), nextSub.getKey(), + nextSub.getValue()); + } + } + } + } + finally { + this.lock.readLock().unlock(); + } + } + catch (InterruptedException e) { + log.error("printAllPeriodically InterruptedException", e); + } + } +} diff --git a/rocketmq-namesrv/src/main/java/com/alibaba/rocketmq/namesrv/kvconfig/KVConfigSerializeWrapper.java b/rocketmq-namesrv/src/main/java/com/alibaba/rocketmq/namesrv/kvconfig/KVConfigSerializeWrapper.java new file mode 100644 index 000000000..605ea68c7 --- /dev/null +++ b/rocketmq-namesrv/src/main/java/com/alibaba/rocketmq/namesrv/kvconfig/KVConfigSerializeWrapper.java @@ -0,0 +1,41 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.namesrv.kvconfig; + +import java.util.HashMap; + +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; + + +/** + * KV配置序列化，json包装 + * + * @author shijia.wxr + * @since 2013-7-25 + */ +public class KVConfigSerializeWrapper extends RemotingSerializable { + private HashMap> configTable; + + + public HashMap> getConfigTable() { + return configTable; + } + + + public void setConfigTable(HashMap> configTable) { + this.configTable = configTable; + } +} diff --git a/rocketmq-namesrv/src/main/java/com/alibaba/rocketmq/namesrv/processor/DefaultRequestProcessor.java b/rocketmq-namesrv/src/main/java/com/alibaba/rocketmq/namesrv/processor/DefaultRequestProcessor.java new file mode 100644 index 000000000..bb274e01b --- /dev/null +++ b/rocketmq-namesrv/src/main/java/com/alibaba/rocketmq/namesrv/processor/DefaultRequestProcessor.java @@ -0,0 +1,590 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.namesrv.processor; + +import io.netty.channel.ChannelHandlerContext; + +import java.util.concurrent.atomic.AtomicLong; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.rocketmq.common.MQVersion; +import com.alibaba.rocketmq.common.MQVersion.Version; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.common.help.FAQUrl; +import com.alibaba.rocketmq.common.namesrv.NamesrvUtil; +import com.alibaba.rocketmq.common.namesrv.RegisterBrokerResult; +import com.alibaba.rocketmq.common.protocol.RequestCode; +import com.alibaba.rocketmq.common.protocol.ResponseCode; +import com.alibaba.rocketmq.common.protocol.body.RegisterBrokerBody; +import com.alibaba.rocketmq.common.protocol.body.TopicConfigSerializeWrapper; +import com.alibaba.rocketmq.common.protocol.header.GetTopicsByClusterRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.namesrv.DeleteKVConfigRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.namesrv.DeleteTopicInNamesrvRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.namesrv.GetKVConfigRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.namesrv.GetKVConfigResponseHeader; +import com.alibaba.rocketmq.common.protocol.header.namesrv.GetKVListByNamespaceRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.namesrv.GetRouteInfoRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.namesrv.PutKVConfigRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.namesrv.RegisterBrokerRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.namesrv.RegisterBrokerResponseHeader; +import com.alibaba.rocketmq.common.protocol.header.namesrv.UnRegisterBrokerRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.namesrv.WipeWritePermOfBrokerRequestHeader; +import com.alibaba.rocketmq.common.protocol.header.namesrv.WipeWritePermOfBrokerResponseHeader; +import com.alibaba.rocketmq.common.protocol.route.TopicRouteData; +import com.alibaba.rocketmq.namesrv.NamesrvController; +import com.alibaba.rocketmq.remoting.common.RemotingHelper; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; +import com.alibaba.rocketmq.remoting.netty.NettyRequestProcessor; +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; + + +/** + * Name Server网络请求处理 + * + * @author shijia.wxr + * @since 2013-7-5 + */ +public class DefaultRequestProcessor implements NettyRequestProcessor { + private static final Logger log = LoggerFactory.getLogger(LoggerName.NamesrvLoggerName); + + private final NamesrvController namesrvController; + + + public DefaultRequestProcessor(NamesrvController namesrvController) { + this.namesrvController = namesrvController; + } + + + @Override + public RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + if (log.isDebugEnabled()) { + log.debug("receive request, {} {} {}",// + request.getCode(), // + RemotingHelper.parseChannelRemoteAddr(ctx.channel()), // + request); + } + + switch (request.getCode()) { + case RequestCode.PUT_KV_CONFIG: + return this.putKVConfig(ctx, request); + case RequestCode.GET_KV_CONFIG: + return this.getKVConfig(ctx, request); + case RequestCode.DELETE_KV_CONFIG: + return this.deleteKVConfig(ctx, request); + case RequestCode.REGISTER_BROKER: + Version brokerVersion = MQVersion.value2Version(request.getVersion()); + // 新版本Broker，支持Filter Server + if (brokerVersion.ordinal() >= MQVersion.Version.V3_0_11.ordinal()) { + return this.registerBrokerWithFilterServer(ctx, request); + } + // 低版本Broker，不支持Filter Server + else { + return this.registerBroker(ctx, request); + } + case RequestCode.UNREGISTER_BROKER: + return this.unregisterBroker(ctx, request); + case RequestCode.GET_ROUTEINTO_BY_TOPIC: + return this.getRouteInfoByTopic(ctx, request); + case RequestCode.GET_BROKER_CLUSTER_INFO: + return this.getBrokerClusterInfo(ctx, request); + case RequestCode.WIPE_WRITE_PERM_OF_BROKER: + return this.wipeWritePermOfBroker(ctx, request); + case RequestCode.GET_ALL_TOPIC_LIST_FROM_NAMESERVER: + return getAllTopicListFromNameserver(ctx, request); + case RequestCode.DELETE_TOPIC_IN_NAMESRV: + return deleteTopicInNamesrv(ctx, request); + case RequestCode.GET_KV_CONFIG_BY_VALUE: + return getKVConfigByValue(ctx, request); + case RequestCode.DELETE_KV_CONFIG_BY_VALUE: + return deleteKVConfigByValue(ctx, request); + case RequestCode.GET_KVLIST_BY_NAMESPACE: + return this.getKVListByNamespace(ctx, request); + case RequestCode.GET_TOPICS_BY_CLUSTER: + return this.getTopicsByCluster(ctx, request); + case RequestCode.GET_SYSTEM_TOPIC_LIST_FROM_NS: + return this.getSystemTopicListFromNs(ctx, request); + case RequestCode.GET_UNIT_TOPIC_LIST: + return this.getUnitTopicList(ctx, request); + case RequestCode.GET_HAS_UNIT_SUB_TOPIC_LIST: + return this.getHasUnitSubTopicList(ctx, request); + case RequestCode.GET_HAS_UNIT_SUB_UNUNIT_TOPIC_LIST: + return this.getHasUnitSubUnUnitTopicList(ctx, request); + default: + break; + } + return null; + } + + + public RemotingCommand registerBrokerWithFilterServer(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + final RemotingCommand response = + RemotingCommand.createResponseCommand(RegisterBrokerResponseHeader.class); + final RegisterBrokerResponseHeader responseHeader = + (RegisterBrokerResponseHeader) response.readCustomHeader(); + final RegisterBrokerRequestHeader requestHeader = + (RegisterBrokerRequestHeader) request + .decodeCommandCustomHeader(RegisterBrokerRequestHeader.class); + + RegisterBrokerBody registerBrokerBody = new RegisterBrokerBody(); + + if (request.getBody() != null) { + registerBrokerBody = RegisterBrokerBody.decode(request.getBody(), RegisterBrokerBody.class); + } + else { + registerBrokerBody.getTopicConfigSerializeWrapper().getDataVersion() + .setCounter(new AtomicLong(0)); + registerBrokerBody.getTopicConfigSerializeWrapper().getDataVersion().setTimestatmp(0); + } + + RegisterBrokerResult result = this.namesrvController.getRouteInfoManager().registerBroker(// + requestHeader.getClusterName(), // 1 + requestHeader.getBrokerAddr(), // 2 + requestHeader.getBrokerName(), // 3 + requestHeader.getBrokerId(), // 4 + requestHeader.getHaServerAddr(),// 5 + registerBrokerBody.getTopicConfigSerializeWrapper(), // 6 + registerBrokerBody.getFilterServerList(),// + ctx.channel()// 7 + ); + + responseHeader.setHaServerAddr(result.getHaServerAddr()); + responseHeader.setMasterAddr(result.getMasterAddr()); + + // 获取顺序消息 topic 列表 + byte[] jsonValue = + this.namesrvController.getKvConfigManager().getKVListByNamespace( + NamesrvUtil.NAMESPACE_ORDER_TOPIC_CONFIG); + response.setBody(jsonValue); + + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + + /** + * 获取一个Namespace下的所有kv + */ + private RemotingCommand getKVListByNamespace(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + final GetKVListByNamespaceRequestHeader requestHeader = + (GetKVListByNamespaceRequestHeader) request + .decodeCommandCustomHeader(GetKVListByNamespaceRequestHeader.class); + + byte[] jsonValue = this.namesrvController.getKvConfigManager().getKVListByNamespace(// + requestHeader.getNamespace()); + if (null != jsonValue) { + response.setBody(jsonValue); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + response.setCode(ResponseCode.QUERY_NOT_FOUND); + response.setRemark("No config item, Namespace: " + requestHeader.getNamespace()); + return response; + } + + + private RemotingCommand deleteTopicInNamesrv(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + final DeleteTopicInNamesrvRequestHeader requestHeader = + (DeleteTopicInNamesrvRequestHeader) request + .decodeCommandCustomHeader(DeleteTopicInNamesrvRequestHeader.class); + + this.namesrvController.getRouteInfoManager().deleteTopic(requestHeader.getTopic()); + + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + + /** + * 获取全部Topic列表 + * + * @param ctx + * @param request + * @return + */ + private RemotingCommand getAllTopicListFromNameserver(ChannelHandlerContext ctx, RemotingCommand request) { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + + byte[] body = this.namesrvController.getRouteInfoManager().getAllTopicList(); + + response.setBody(body); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + + private RemotingCommand wipeWritePermOfBroker(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + final RemotingCommand response = + RemotingCommand.createResponseCommand(WipeWritePermOfBrokerResponseHeader.class); + final WipeWritePermOfBrokerResponseHeader responseHeader = + (WipeWritePermOfBrokerResponseHeader) response.readCustomHeader(); + final WipeWritePermOfBrokerRequestHeader requestHeader = + (WipeWritePermOfBrokerRequestHeader) request + .decodeCommandCustomHeader(WipeWritePermOfBrokerRequestHeader.class); + + int wipeTopicCnt = + this.namesrvController.getRouteInfoManager().wipeWritePermOfBrokerByLock( + requestHeader.getBrokerName()); + + log.info("wipe write perm of broker[{}], client: {}, {}", // + requestHeader.getBrokerName(), // + RemotingHelper.parseChannelRemoteAddr(ctx.channel()), // + wipeTopicCnt); + + responseHeader.setWipeTopicCount(wipeTopicCnt); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + + private RemotingCommand getBrokerClusterInfo(ChannelHandlerContext ctx, RemotingCommand request) { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + + byte[] content = this.namesrvController.getRouteInfoManager().getAllClusterInfo(); + response.setBody(content); + + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + + public RemotingCommand getRouteInfoByTopic(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + final GetRouteInfoRequestHeader requestHeader = + (GetRouteInfoRequestHeader) request + .decodeCommandCustomHeader(GetRouteInfoRequestHeader.class); + + TopicRouteData topicRouteData = + this.namesrvController.getRouteInfoManager().pickupTopicRouteData(requestHeader.getTopic()); + + if (topicRouteData != null) { + String orderTopicConf = + this.namesrvController.getKvConfigManager().getKVConfig( + NamesrvUtil.NAMESPACE_ORDER_TOPIC_CONFIG, requestHeader.getTopic()); + topicRouteData.setOrderTopicConf(orderTopicConf); + + byte[] content = topicRouteData.encode(); + response.setBody(content); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + response.setCode(ResponseCode.TOPIC_NOT_EXIST); + response.setRemark("No topic route info in name server for the topic: " + requestHeader.getTopic() + + FAQUrl.suggestTodo(FAQUrl.APPLY_TOPIC_URL)); + return response; + } + + + public RemotingCommand putKVConfig(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + final PutKVConfigRequestHeader requestHeader = + (PutKVConfigRequestHeader) request.decodeCommandCustomHeader(PutKVConfigRequestHeader.class); + + this.namesrvController.getKvConfigManager().putKVConfig(// + requestHeader.getNamespace(),// + requestHeader.getKey(),// + requestHeader.getValue()// + ); + + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + + public RemotingCommand getKVConfig(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + final RemotingCommand response = + RemotingCommand.createResponseCommand(GetKVConfigResponseHeader.class); + final GetKVConfigResponseHeader responseHeader = + (GetKVConfigResponseHeader) response.readCustomHeader(); + final GetKVConfigRequestHeader requestHeader = + (GetKVConfigRequestHeader) request.decodeCommandCustomHeader(GetKVConfigRequestHeader.class); + + String value = this.namesrvController.getKvConfigManager().getKVConfig(// + requestHeader.getNamespace(),// + requestHeader.getKey()// + ); + + if (value != null) { + responseHeader.setValue(value); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + response.setCode(ResponseCode.QUERY_NOT_FOUND); + response.setRemark("No config item, Namespace: " + requestHeader.getNamespace() + " Key: " + + requestHeader.getKey()); + return response; + } + + + public RemotingCommand deleteKVConfig(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + final DeleteKVConfigRequestHeader requestHeader = + (DeleteKVConfigRequestHeader) request + .decodeCommandCustomHeader(DeleteKVConfigRequestHeader.class); + + this.namesrvController.getKvConfigManager().deleteKVConfig(// + requestHeader.getNamespace(),// + requestHeader.getKey()// + ); + + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + + public RemotingCommand registerBroker(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + final RemotingCommand response = + RemotingCommand.createResponseCommand(RegisterBrokerResponseHeader.class); + final RegisterBrokerResponseHeader responseHeader = + (RegisterBrokerResponseHeader) response.readCustomHeader(); + final RegisterBrokerRequestHeader requestHeader = + (RegisterBrokerRequestHeader) request + .decodeCommandCustomHeader(RegisterBrokerRequestHeader.class); + + TopicConfigSerializeWrapper topicConfigWrapper = null; + if (request.getBody() != null) { + topicConfigWrapper = + TopicConfigSerializeWrapper.decode(request.getBody(), TopicConfigSerializeWrapper.class); + } + else { + topicConfigWrapper = new TopicConfigSerializeWrapper(); + topicConfigWrapper.getDataVersion().setCounter(new AtomicLong(0)); + topicConfigWrapper.getDataVersion().setTimestatmp(0); + } + + RegisterBrokerResult result = this.namesrvController.getRouteInfoManager().registerBroker(// + requestHeader.getClusterName(), // 1 + requestHeader.getBrokerAddr(), // 2 + requestHeader.getBrokerName(), // 3 + requestHeader.getBrokerId(), // 4 + requestHeader.getHaServerAddr(),// 5 + topicConfigWrapper, // 6 + null,// + ctx.channel()// 7 + ); + + responseHeader.setHaServerAddr(result.getHaServerAddr()); + responseHeader.setMasterAddr(result.getMasterAddr()); + + // 获取顺序消息 topic 列表 + byte[] jsonValue = + this.namesrvController.getKvConfigManager().getKVListByNamespace( + NamesrvUtil.NAMESPACE_ORDER_TOPIC_CONFIG); + response.setBody(jsonValue); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + + public RemotingCommand unregisterBroker(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + final UnRegisterBrokerRequestHeader requestHeader = + (UnRegisterBrokerRequestHeader) request + .decodeCommandCustomHeader(UnRegisterBrokerRequestHeader.class); + + this.namesrvController.getRouteInfoManager().unregisterBroker(// + requestHeader.getClusterName(), // 1 + requestHeader.getBrokerAddr(), // 2 + requestHeader.getBrokerName(), // 3 + requestHeader.getBrokerId()); + + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + + public RemotingCommand getKVConfigByValue(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + final RemotingCommand response = + RemotingCommand.createResponseCommand(GetKVConfigResponseHeader.class); + final GetKVConfigResponseHeader responseHeader = + (GetKVConfigResponseHeader) response.readCustomHeader(); + final GetKVConfigRequestHeader requestHeader = + (GetKVConfigRequestHeader) request.decodeCommandCustomHeader(GetKVConfigRequestHeader.class); + + String value = this.namesrvController.getKvConfigManager().getKVConfigByValue(// + requestHeader.getNamespace(),// + requestHeader.getKey()// + ); + + if (value != null) { + responseHeader.setValue(value); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + response.setCode(ResponseCode.QUERY_NOT_FOUND); + response.setRemark("No config item, Namespace: " + requestHeader.getNamespace() + " Key: " + + requestHeader.getKey()); + return response; + } + + + public RemotingCommand deleteKVConfigByValue(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + final DeleteKVConfigRequestHeader requestHeader = + (DeleteKVConfigRequestHeader) request + .decodeCommandCustomHeader(DeleteKVConfigRequestHeader.class); + + this.namesrvController.getKvConfigManager().deleteKVConfigByValue(// + requestHeader.getNamespace(),// + requestHeader.getKey()// + ); + + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + + /** + * 获取指定集群下的全部Topic列表 + * + * @param ctx + * @param request + * @return + */ + private RemotingCommand getTopicsByCluster(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + final GetTopicsByClusterRequestHeader requestHeader = + (GetTopicsByClusterRequestHeader) request + .decodeCommandCustomHeader(GetTopicsByClusterRequestHeader.class); + + byte[] body = + this.namesrvController.getRouteInfoManager().getTopicsByCluster(requestHeader.getCluster()); + + response.setBody(body); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + + /** + * 获取所有系统内置 Topic 列表 + * + * @param ctx + * @param request + * @return + * @throws RemotingCommandException + */ + private RemotingCommand getSystemTopicListFromNs(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + + byte[] body = this.namesrvController.getRouteInfoManager().getSystemTopicList(); + + response.setBody(body); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + + /** + * 获取单元化逻辑 Topic 列表 + * + * @param ctx + * @param request + * @return + * @throws RemotingCommandException + */ + private RemotingCommand getUnitTopicList(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + + byte[] body = this.namesrvController.getRouteInfoManager().getUnitTopics(); + + response.setBody(body); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + + /** + * 获取含有单元化订阅组的 Topic 列表 + * + * @param ctx + * @param request + * @return + * @throws RemotingCommandException + */ + private RemotingCommand getHasUnitSubTopicList(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + + byte[] body = this.namesrvController.getRouteInfoManager().getHasUnitSubTopicList(); + + response.setBody(body); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } + + + /** + * 获取含有单元化订阅组的非单元化 Topic 列表 + * + * @param ctx + * @param request + * @return + * @throws RemotingCommandException + */ + private RemotingCommand getHasUnitSubUnUnitTopicList(ChannelHandlerContext ctx, RemotingCommand request) + throws RemotingCommandException { + final RemotingCommand response = RemotingCommand.createResponseCommand(null); + + byte[] body = this.namesrvController.getRouteInfoManager().getHasUnitSubUnUnitTopicList(); + + response.setBody(body); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; + } +} diff --git a/rocketmq-namesrv/src/main/java/com/alibaba/rocketmq/namesrv/routeinfo/BrokerHousekeepingService.java b/rocketmq-namesrv/src/main/java/com/alibaba/rocketmq/namesrv/routeinfo/BrokerHousekeepingService.java new file mode 100644 index 000000000..94b4fc88a --- /dev/null +++ b/rocketmq-namesrv/src/main/java/com/alibaba/rocketmq/namesrv/routeinfo/BrokerHousekeepingService.java @@ -0,0 +1,63 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.namesrv.routeinfo; + +import io.netty.channel.Channel; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.namesrv.NamesrvController; +import com.alibaba.rocketmq.remoting.ChannelEventListener; + + +/** + * @author shijia.wxr + * @since 2013-7-15 + */ +public class BrokerHousekeepingService implements ChannelEventListener { + private static final Logger log = LoggerFactory.getLogger(LoggerName.NamesrvLoggerName); + private final NamesrvController namesrvController; + + + public BrokerHousekeepingService(NamesrvController namesrvController) { + this.namesrvController = namesrvController; + } + + + @Override + public void onChannelConnect(String remoteAddr, Channel channel) { + } + + + @Override + public void onChannelClose(String remoteAddr, Channel channel) { + this.namesrvController.getRouteInfoManager().onChannelDestroy(remoteAddr, channel); + } + + + @Override + public void onChannelException(String remoteAddr, Channel channel) { + this.namesrvController.getRouteInfoManager().onChannelDestroy(remoteAddr, channel); + } + + + @Override + public void onChannelIdle(String remoteAddr, Channel channel) { + this.namesrvController.getRouteInfoManager().onChannelDestroy(remoteAddr, channel); + } +} diff --git a/rocketmq-namesrv/src/main/java/com/alibaba/rocketmq/namesrv/routeinfo/RouteInfoManager.java b/rocketmq-namesrv/src/main/java/com/alibaba/rocketmq/namesrv/routeinfo/RouteInfoManager.java new file mode 100644 index 000000000..ab59d084b --- /dev/null +++ b/rocketmq-namesrv/src/main/java/com/alibaba/rocketmq/namesrv/routeinfo/RouteInfoManager.java @@ -0,0 +1,939 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.namesrv.routeinfo; + +import io.netty.channel.Channel; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map.Entry; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.rocketmq.common.DataVersion; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.TopicConfig; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.common.constant.PermName; +import com.alibaba.rocketmq.common.namesrv.RegisterBrokerResult; +import com.alibaba.rocketmq.common.protocol.body.ClusterInfo; +import com.alibaba.rocketmq.common.protocol.body.TopicConfigSerializeWrapper; +import com.alibaba.rocketmq.common.protocol.body.TopicList; +import com.alibaba.rocketmq.common.protocol.route.BrokerData; +import com.alibaba.rocketmq.common.protocol.route.QueueData; +import com.alibaba.rocketmq.common.protocol.route.TopicRouteData; +import com.alibaba.rocketmq.common.sysflag.TopicSysFlag; +import com.alibaba.rocketmq.remoting.common.RemotingUtil; + + +/** + * 运行过程中的路由信息，数据只在内存，宕机后数据消失，但是Broker会定期推送最新数据 + * + * @author shijia.wxr + * @since 2013-7-2 + */ +public class RouteInfoManager { + private static final Logger log = LoggerFactory.getLogger(LoggerName.NamesrvLoggerName); + private final ReadWriteLock lock = new ReentrantReadWriteLock(); + private final HashMap> topicQueueTable; + private final HashMap brokerAddrTable; + private final HashMap> clusterAddrTable; + private final HashMap brokerLiveTable; + private final HashMap/* Filter Server */> filterServerTable; + + + public RouteInfoManager() { + this.topicQueueTable = new HashMap>(1024); + this.brokerAddrTable = new HashMap(128); + this.clusterAddrTable = new HashMap>(32); + this.brokerLiveTable = new HashMap(256); + this.filterServerTable = new HashMap>(256); + } + + + public byte[] getAllClusterInfo() { + ClusterInfo clusterInfoSerializeWrapper = new ClusterInfo(); + clusterInfoSerializeWrapper.setBrokerAddrTable(this.brokerAddrTable); + clusterInfoSerializeWrapper.setClusterAddrTable(this.clusterAddrTable); + return clusterInfoSerializeWrapper.encode(); + } + + + public void deleteTopic(final String topic) { + try { + try { + this.lock.writeLock().lockInterruptibly(); + this.topicQueueTable.remove(topic); + } + finally { + this.lock.writeLock().unlock(); + } + } + catch (Exception e) { + log.error("deleteTopic Exception", e); + } + } + + + public byte[] getAllTopicList() { + TopicList topicList = new TopicList(); + try { + try { + this.lock.readLock().lockInterruptibly(); + topicList.getTopicList().addAll(this.topicQueueTable.keySet()); + } + finally { + this.lock.readLock().unlock(); + } + } + catch (Exception e) { + log.error("getAllTopicList Exception", e); + } + + return topicList.encode(); + } + + + /** + * @return 如果是slave，则返回master的ha地址 + */ + public RegisterBrokerResult registerBroker(// + final String clusterName,// 1 + final String brokerAddr,// 2 + final String brokerName,// 3 + final long brokerId,// 4 + final String haServerAddr,// 5 + final TopicConfigSerializeWrapper topicConfigWrapper,// 6 + final List filterServerList, // 7 + final Channel channel// 8 + ) { + RegisterBrokerResult result = new RegisterBrokerResult(); + try { + try { + this.lock.writeLock().lockInterruptibly(); + + // 更新集群信息 + Set brokerNames = this.clusterAddrTable.get(clusterName); + if (null == brokerNames) { + brokerNames = new HashSet(); + this.clusterAddrTable.put(clusterName, brokerNames); + } + brokerNames.add(brokerName); + + boolean registerFirst = false; + + // 更新主备信息 + BrokerData brokerData = this.brokerAddrTable.get(brokerName); + if (null == brokerData) { + registerFirst = true; + brokerData = new BrokerData(); + brokerData.setBrokerName(brokerName); + HashMap brokerAddrs = new HashMap(); + brokerData.setBrokerAddrs(brokerAddrs); + + this.brokerAddrTable.put(brokerName, brokerData); + } + String oldAddr = brokerData.getBrokerAddrs().put(brokerId, brokerAddr); + registerFirst = registerFirst || (null == oldAddr); + + // 更新Topic信息 + if (null != topicConfigWrapper // + && MixAll.MASTER_ID == brokerId) { + if (this.isBrokerTopicConfigChanged(brokerAddr, topicConfigWrapper.getDataVersion())// + || registerFirst) { + ConcurrentHashMap tcTable = + topicConfigWrapper.getTopicConfigTable(); + if (tcTable != null) { + for (String topic : tcTable.keySet()) { + TopicConfig topicConfig = tcTable.get(topic); + this.createAndUpdateQueueData(brokerName, topicConfig); + } + } + } + } + + // 更新最后变更时间 + BrokerLiveInfo prevBrokerLiveInfo = this.brokerLiveTable.put(brokerAddr, // + new BrokerLiveInfo(// + System.currentTimeMillis(), // + topicConfigWrapper.getDataVersion(),// + channel, // + haServerAddr)); + if (null == prevBrokerLiveInfo) { + log.info("new broker registerd, {} HAServer: {}", brokerAddr, haServerAddr); + } + + // 更新Filter Server列表 + if (filterServerList != null) { + if (filterServerList.isEmpty()) { + this.filterServerTable.remove(brokerAddr); + } + else { + this.filterServerTable.put(brokerAddr, filterServerList); + } + } + + // 返回值 + if (MixAll.MASTER_ID != brokerId) { + String masterAddr = brokerData.getBrokerAddrs().get(MixAll.MASTER_ID); + if (masterAddr != null) { + BrokerLiveInfo brokerLiveInfo = this.brokerLiveTable.get(masterAddr); + if (brokerLiveInfo != null) { + result.setHaServerAddr(brokerLiveInfo.getHaServerAddr()); + result.setMasterAddr(masterAddr); + } + } + } + } + finally { + this.lock.writeLock().unlock(); + } + } + catch (Exception e) { + log.error("registerBroker Exception", e); + } + + return result; + } + + + /** + * 判断Topic配置信息是否发生变更 + */ + private boolean isBrokerTopicConfigChanged(final String brokerAddr, final DataVersion dataVersion) { + BrokerLiveInfo prev = this.brokerLiveTable.get(brokerAddr); + if (null == prev || !prev.getDataVersion().equals(dataVersion)) { + return true; + } + + return false; + } + + + public int wipeWritePermOfBrokerByLock(final String brokerName) { + try { + try { + this.lock.writeLock().lockInterruptibly(); + return wipeWritePermOfBroker(brokerName); + } + finally { + this.lock.writeLock().unlock(); + } + } + catch (Exception e) { + log.error("wipeWritePermOfBrokerByLock Exception", e); + } + + return 0; + } + + + private int wipeWritePermOfBroker(final String brokerName) { + int wipeTopicCnt = 0; + Iterator>> itTopic = this.topicQueueTable.entrySet().iterator(); + while (itTopic.hasNext()) { + Entry> entry = itTopic.next(); + List qdList = entry.getValue(); + + Iterator it = qdList.iterator(); + while (it.hasNext()) { + QueueData qd = it.next(); + if (qd.getBrokerName().equals(brokerName)) { + int perm = qd.getPerm(); + perm &= ~PermName.PERM_WRITE; + qd.setPerm(perm); + wipeTopicCnt++; + } + } + } + + return wipeTopicCnt; + } + + + private void createAndUpdateQueueData(final String brokerName, final TopicConfig topicConfig) { + QueueData queueData = new QueueData(); + queueData.setBrokerName(brokerName); + queueData.setWriteQueueNums(topicConfig.getWriteQueueNums()); + queueData.setReadQueueNums(topicConfig.getReadQueueNums()); + queueData.setPerm(topicConfig.getPerm()); + queueData.setTopicSynFlag(topicConfig.getTopicSysFlag()); + + List queueDataList = this.topicQueueTable.get(topicConfig.getTopicName()); + if (null == queueDataList) { + queueDataList = new LinkedList(); + queueDataList.add(queueData); + this.topicQueueTable.put(topicConfig.getTopicName(), queueDataList); + log.info("new topic registerd, {} {}", topicConfig.getTopicName(), queueData); + } + else { + boolean addNewOne = true; + + Iterator it = queueDataList.iterator(); + while (it.hasNext()) { + QueueData qd = it.next(); + if (qd.getBrokerName().equals(brokerName)) { + if (qd.equals(queueData)) { + addNewOne = false; + } + else { + log.info("topic changed, {} OLD: {} NEW: {}", topicConfig.getTopicName(), qd, + queueData); + it.remove(); + } + } + } + + if (addNewOne) { + queueDataList.add(queueData); + } + } + } + + + public void unregisterBroker(// + final String clusterName,// 1 + final String brokerAddr,// 2 + final String brokerName,// 3 + final long brokerId// 4 + ) { + try { + try { + this.lock.writeLock().lockInterruptibly(); + BrokerLiveInfo brokerLiveInfo = this.brokerLiveTable.remove(brokerAddr); + if (brokerLiveInfo != null) { + log.info("unregisterBroker, remove from brokerLiveTable {}, {}", // + (brokerLiveInfo != null ? "OK" : "Failed"),// + brokerAddr// + ); + } + + this.filterServerTable.remove(brokerAddr); + + boolean removeBrokerName = false; + BrokerData brokerData = this.brokerAddrTable.get(brokerName); + if (null != brokerData) { + String addr = brokerData.getBrokerAddrs().remove(brokerId); + log.info("unregisterBroker, remove addr from brokerAddrTable {}, {}", // + (addr != null ? "OK" : "Failed"),// + brokerAddr// + ); + + if (brokerData.getBrokerAddrs().isEmpty()) { + this.brokerAddrTable.remove(brokerName); + log.info("unregisterBroker, remove name from brokerAddrTable OK, {}", // + brokerName// + ); + + removeBrokerName = true; + } + } + + if (removeBrokerName) { + Set nameSet = this.clusterAddrTable.get(clusterName); + if (nameSet != null) { + boolean removed = nameSet.remove(brokerName); + log.info("unregisterBroker, remove name from clusterAddrTable {}, {}", // + (removed ? "OK" : "Failed"),// + brokerName// + ); + + if (nameSet.isEmpty()) { + this.clusterAddrTable.remove(clusterName); + log.info("unregisterBroker, remove cluster from clusterAddrTable {}", // + clusterName// + ); + } + } + + // 删除相应的topic + this.removeTopicByBrokerName(brokerName); + } + } + finally { + this.lock.writeLock().unlock(); + } + } + catch (Exception e) { + log.error("unregisterBroker Exception", e); + } + } + + + private void removeTopicByBrokerName(final String brokerName) { + Iterator>> itMap = this.topicQueueTable.entrySet().iterator(); + while (itMap.hasNext()) { + Entry> entry = itMap.next(); + + String topic = entry.getKey(); + List queueDataList = entry.getValue(); + Iterator it = queueDataList.iterator(); + while (it.hasNext()) { + QueueData qd = it.next(); + if (qd.getBrokerName().equals(brokerName)) { + log.info("removeTopicByBrokerName, remove one broker's topic {} {}", topic, qd); + it.remove(); + } + } + + if (queueDataList.isEmpty()) { + log.info("removeTopicByBrokerName, remove the topic all queue {}", topic); + itMap.remove(); + } + } + } + + + public TopicRouteData pickupTopicRouteData(final String topic) { + TopicRouteData topicRouteData = new TopicRouteData(); + boolean foundQueueData = false; + boolean foundBrokerData = false; + Set brokerNameSet = new HashSet(); + List brokerDataList = new LinkedList(); + topicRouteData.setBrokerDatas(brokerDataList); + + HashMap> filterServerMap = new HashMap>(); + topicRouteData.setFilterServerTable(filterServerMap); + + try { + try { + this.lock.readLock().lockInterruptibly(); + List queueDataList = this.topicQueueTable.get(topic); + if (queueDataList != null) { + topicRouteData.setQueueDatas(queueDataList); + foundQueueData = true; + + // BrokerName去重 + Iterator it = queueDataList.iterator(); + while (it.hasNext()) { + QueueData qd = it.next(); + brokerNameSet.add(qd.getBrokerName()); + } + + for (String brokerName : brokerNameSet) { + BrokerData brokerData = this.brokerAddrTable.get(brokerName); + if (null != brokerData) { + BrokerData brokerDataClone = new BrokerData(); + brokerDataClone.setBrokerName(brokerData.getBrokerName()); + brokerDataClone.setBrokerAddrs((HashMap) brokerData + .getBrokerAddrs().clone()); + brokerDataList.add(brokerDataClone); + foundBrokerData = true; + + // 增加Filter Server + for (final String brokerAddr : brokerDataClone.getBrokerAddrs().values()) { + List filterServerList = this.filterServerTable.get(brokerAddr); + filterServerMap.put(brokerAddr, filterServerList); + } + } + } + } + } + finally { + this.lock.readLock().unlock(); + } + } + catch (Exception e) { + log.error("pickupTopicRouteData Exception", e); + } + + if (log.isDebugEnabled()) { + log.debug("pickupTopicRouteData {} {}", topic, topicRouteData); + } + + if (foundBrokerData && foundQueueData) { + return topicRouteData; + } + + return null; + } + + // Broker Channel两分钟过期 + private final static long BrokerChannelExpiredTime = 1000 * 60 * 2; + + + public void scanNotActiveBroker() { + Iterator> it = this.brokerLiveTable.entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + long last = next.getValue().getLastUpdateTimestamp(); + if ((last + BrokerChannelExpiredTime) < System.currentTimeMillis()) { + RemotingUtil.closeChannel(next.getValue().getChannel()); + it.remove(); + log.warn("The broker channel expired, {} {}ms", next.getKey(), BrokerChannelExpiredTime); + this.onChannelDestroy(next.getKey(), next.getValue().getChannel()); + } + } + } + + + /** + * Channel被关闭，或者Channel Idle时间超限 + */ + public void onChannelDestroy(String remoteAddr, Channel channel) { + String brokerAddrFound = null; + + // 加读锁，寻找断开连接的Broker + if (channel != null) { + try { + try { + this.lock.readLock().lockInterruptibly(); + Iterator> itBrokerLiveTable = + this.brokerLiveTable.entrySet().iterator(); + while (itBrokerLiveTable.hasNext()) { + Entry entry = itBrokerLiveTable.next(); + if (entry.getValue().getChannel() == channel) { + brokerAddrFound = entry.getKey(); + break; + } + } + } + finally { + this.lock.readLock().unlock(); + } + } + catch (Exception e) { + log.error("onChannelDestroy Exception", e); + } + } + + if (null == brokerAddrFound) { + brokerAddrFound = remoteAddr; + } + else { + log.info("the broker's channel destroyed, {}, clean it's data structure at once", brokerAddrFound); + } + + // 加写锁，删除相关数据结构 + if (brokerAddrFound != null && brokerAddrFound.length() > 0) { + + try { + try { + this.lock.writeLock().lockInterruptibly(); + // 清理brokerLiveTable + this.brokerLiveTable.remove(brokerAddrFound); + + // 清理Filter Server + this.filterServerTable.remove(brokerAddrFound); + + // 清理brokerAddrTable + String brokerNameFound = null; + boolean removeBrokerName = false; + Iterator> itBrokerAddrTable = + this.brokerAddrTable.entrySet().iterator(); + while (itBrokerAddrTable.hasNext() && (null == brokerNameFound)) { + BrokerData brokerData = itBrokerAddrTable.next().getValue(); + + // 遍历Master/Slave，删除brokerAddr + Iterator> it = brokerData.getBrokerAddrs().entrySet().iterator(); + while (it.hasNext()) { + Entry entry = it.next(); + Long brokerId = entry.getKey(); + String brokerAddr = entry.getValue(); + if (brokerAddr.equals(brokerAddrFound)) { + brokerNameFound = brokerData.getBrokerName(); + it.remove(); + log.info( + "remove brokerAddr[{}, {}] from brokerAddrTable, because channel destroyed", + brokerId, brokerAddr); + break; + } + } + + // BrokerName无关联BrokerAddr + if (brokerData.getBrokerAddrs().isEmpty()) { + removeBrokerName = true; + itBrokerAddrTable.remove(); + log.info("remove brokerName[{}] from brokerAddrTable, because channel destroyed", + brokerData.getBrokerName()); + } + } + + // 清理clusterAddrTable + if (brokerNameFound != null && removeBrokerName) { + Iterator>> it = this.clusterAddrTable.entrySet().iterator(); + while (it.hasNext()) { + Entry> entry = it.next(); + String clusterName = entry.getKey(); + Set brokerNames = entry.getValue(); + boolean removed = brokerNames.remove(brokerNameFound); + if (removed) { + log.info( + "remove brokerName[{}], clusterName[{}] from clusterAddrTable, because channel destroyed", + brokerNameFound, clusterName); + + // 如果集群对应的所有broker都下线了， 则集群也删除掉 + if (brokerNames.isEmpty()) { + log.info( + "remove the clusterName[{}] from clusterAddrTable, because channel destroyed and no broker in this cluster", + clusterName); + it.remove(); + } + + break; + } + } + } + + // 清理topicQueueTable + if (removeBrokerName) { + Iterator>> itTopicQueueTable = + this.topicQueueTable.entrySet().iterator(); + while (itTopicQueueTable.hasNext()) { + Entry> entry = itTopicQueueTable.next(); + String topic = entry.getKey(); + List queueDataList = entry.getValue(); + + Iterator itQueueData = queueDataList.iterator(); + while (itQueueData.hasNext()) { + QueueData queueData = itQueueData.next(); + if (queueData.getBrokerName().equals(brokerNameFound)) { + itQueueData.remove(); + log.info( + "remove topic[{} {}], from topicQueueTable, because channel destroyed", + topic, queueData); + } + } + + if (queueDataList.isEmpty()) { + itTopicQueueTable.remove(); + log.info( + "remove topic[{}] all queue, from topicQueueTable, because channel destroyed", + topic); + } + } + } + } + finally { + this.lock.writeLock().unlock(); + } + } + catch (Exception e) { + log.error("onChannelDestroy Exception", e); + } + } + } + + + /** + * 定期打印当前类的数据结构 + */ + public void printAllPeriodically() { + try { + try { + this.lock.readLock().lockInterruptibly(); + log.info("--------------------------------------------------------"); + { + log.info("topicQueueTable SIZE: {}", this.topicQueueTable.size()); + Iterator>> it = this.topicQueueTable.entrySet().iterator(); + while (it.hasNext()) { + Entry> next = it.next(); + log.info("topicQueueTable Topic: {} {}", next.getKey(), next.getValue()); + } + } + + { + log.info("brokerAddrTable SIZE: {}", this.brokerAddrTable.size()); + Iterator> it = this.brokerAddrTable.entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + log.info("brokerAddrTable brokerName: {} {}", next.getKey(), next.getValue()); + } + } + + { + log.info("brokerLiveTable SIZE: {}", this.brokerLiveTable.size()); + Iterator> it = this.brokerLiveTable.entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + log.info("brokerLiveTable brokerAddr: {} {}", next.getKey(), next.getValue()); + } + } + + { + log.info("clusterAddrTable SIZE: {}", this.clusterAddrTable.size()); + Iterator>> it = this.clusterAddrTable.entrySet().iterator(); + while (it.hasNext()) { + Entry> next = it.next(); + log.info("clusterAddrTable clusterName: {} {}", next.getKey(), next.getValue()); + } + } + } + finally { + this.lock.readLock().unlock(); + } + } + catch (Exception e) { + log.error("printAllPeriodically Exception", e); + } + } + + + /** + * 获取指定集群下的所有 topic 列表 + * + * @return + */ + public byte[] getSystemTopicList() { + TopicList topicList = new TopicList(); + try { + try { + this.lock.readLock().lockInterruptibly(); + for (String cluster : clusterAddrTable.keySet()) { + topicList.getTopicList().add(cluster); + topicList.getTopicList().addAll(this.clusterAddrTable.get(cluster)); + } + + // 随机取一台 broker + if (brokerAddrTable != null && !brokerAddrTable.isEmpty()) { + Iterator it = brokerAddrTable.keySet().iterator(); + while (it.hasNext()) { + BrokerData bd = brokerAddrTable.get(it.next()); + HashMap brokerAddrs = bd.getBrokerAddrs(); + if (bd.getBrokerAddrs() != null && !bd.getBrokerAddrs().isEmpty()) { + Iterator it2 = brokerAddrs.keySet().iterator(); + topicList.setBrokerAddr(brokerAddrs.get(it2.next())); + break; + } + } + } + } + finally { + this.lock.readLock().unlock(); + } + } + catch (Exception e) { + log.error("getAllTopicList Exception", e); + } + + return topicList.encode(); + } + + + /** + * 获取指定集群下的所有 topic 列表 + * + * @param cluster + * @return + */ + public byte[] getTopicsByCluster(String cluster) { + TopicList topicList = new TopicList(); + try { + try { + this.lock.readLock().lockInterruptibly(); + Set brokerNameSet = this.clusterAddrTable.get(cluster); + for (String brokerName : brokerNameSet) { + Iterator>> topicTableIt = + this.topicQueueTable.entrySet().iterator(); + while (topicTableIt.hasNext()) { + Entry> topicEntry = topicTableIt.next(); + String topic = topicEntry.getKey(); + List queueDatas = topicEntry.getValue(); + for (QueueData queueData : queueDatas) { + if (brokerName.equals(queueData.getBrokerName())) { + topicList.getTopicList().add(topic); + break; + } + } + } + } + } + finally { + this.lock.readLock().unlock(); + } + } + catch (Exception e) { + log.error("getAllTopicList Exception", e); + } + + return topicList.encode(); + } + + + /** + * 获取单元逻辑下的所有 topic 列表 + * + * @return + */ + public byte[] getUnitTopics() { + TopicList topicList = new TopicList(); + try { + try { + this.lock.readLock().lockInterruptibly(); + Iterator>> topicTableIt = + this.topicQueueTable.entrySet().iterator(); + while (topicTableIt.hasNext()) { + Entry> topicEntry = topicTableIt.next(); + String topic = topicEntry.getKey(); + List queueDatas = topicEntry.getValue(); + if (queueDatas != null && queueDatas.size() > 0 + && TopicSysFlag.hasUnitFlag(queueDatas.get(0).getTopicSynFlag())) { + topicList.getTopicList().add(topic); + } + } + } + finally { + this.lock.readLock().unlock(); + } + } + catch (Exception e) { + log.error("getAllTopicList Exception", e); + } + + return topicList.encode(); + } + + + /** + * 获取中心向单元同步的所有 topic 列表 + * + * @return + */ + public byte[] getHasUnitSubTopicList() { + TopicList topicList = new TopicList(); + try { + try { + this.lock.readLock().lockInterruptibly(); + Iterator>> topicTableIt = + this.topicQueueTable.entrySet().iterator(); + while (topicTableIt.hasNext()) { + Entry> topicEntry = topicTableIt.next(); + String topic = topicEntry.getKey(); + List queueDatas = topicEntry.getValue(); + if (queueDatas != null && queueDatas.size() > 0 + && TopicSysFlag.hasUnitSubFlag(queueDatas.get(0).getTopicSynFlag())) { + topicList.getTopicList().add(topic); + } + } + } + finally { + this.lock.readLock().unlock(); + } + } + catch (Exception e) { + log.error("getAllTopicList Exception", e); + } + + return topicList.encode(); + } + + + /** + * 获取含有单元化订阅组的非单元化 Topic 列表 + * + * @return + */ + public byte[] getHasUnitSubUnUnitTopicList() { + TopicList topicList = new TopicList(); + try { + try { + this.lock.readLock().lockInterruptibly(); + Iterator>> topicTableIt = + this.topicQueueTable.entrySet().iterator(); + while (topicTableIt.hasNext()) { + Entry> topicEntry = topicTableIt.next(); + String topic = topicEntry.getKey(); + List queueDatas = topicEntry.getValue(); + if (queueDatas != null && queueDatas.size() > 0 + && !TopicSysFlag.hasUnitFlag(queueDatas.get(0).getTopicSynFlag()) + && TopicSysFlag.hasUnitSubFlag(queueDatas.get(0).getTopicSynFlag())) { + topicList.getTopicList().add(topic); + } + } + } + finally { + this.lock.readLock().unlock(); + } + } + catch (Exception e) { + log.error("getAllTopicList Exception", e); + } + + return topicList.encode(); + } +} + + +class BrokerLiveInfo { + private long lastUpdateTimestamp; + private DataVersion dataVersion; + private Channel channel; + private String haServerAddr; + + + public BrokerLiveInfo(long lastUpdateTimestamp, DataVersion dataVersion, Channel channel, + String haServerAddr) { + this.lastUpdateTimestamp = lastUpdateTimestamp; + this.dataVersion = dataVersion; + this.channel = channel; + this.haServerAddr = haServerAddr; + } + + + public long getLastUpdateTimestamp() { + return lastUpdateTimestamp; + } + + + public void setLastUpdateTimestamp(long lastUpdateTimestamp) { + this.lastUpdateTimestamp = lastUpdateTimestamp; + } + + + public DataVersion getDataVersion() { + return dataVersion; + } + + + public void setDataVersion(DataVersion dataVersion) { + this.dataVersion = dataVersion; + } + + + public Channel getChannel() { + return channel; + } + + + public void setChannel(Channel channel) { + this.channel = channel; + } + + + public String getHaServerAddr() { + return haServerAddr; + } + + + public void setHaServerAddr(String haServerAddr) { + this.haServerAddr = haServerAddr; + } + + + @Override + public String toString() { + return "BrokerLiveInfo [lastUpdateTimestamp=" + lastUpdateTimestamp + ", dataVersion=" + dataVersion + + ", channel=" + channel + ", haServerAddr=" + haServerAddr + "]"; + } +} diff --git a/rocketmq-remoting/pom.xml b/rocketmq-remoting/pom.xml new file mode 100644 index 000000000..ab2fd6178 --- /dev/null +++ b/rocketmq-remoting/pom.xml @@ -0,0 +1,34 @@ + + + com.alibaba.rocketmq + rocketmq-all + 3.2.2 + + + 4.0.0 + jar + rocketmq-remoting + rocketmq-remoting ${project.version} + + + + + junit + junit + test + + + com.alibaba + fastjson + + + io.netty + netty-all + + + org.slf4j + slf4j-api + + + diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/ChannelEventListener.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/ChannelEventListener.java new file mode 100644 index 000000000..ee678205e --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/ChannelEventListener.java @@ -0,0 +1,38 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.remoting; + +import io.netty.channel.Channel; + + +/** + * 监听Channel的事件，包括连接断开、连接建立、连接异常，传送这些事件到应用层 + * + * @author shijia.wxr + * @since 2013-7-13 + */ +public interface ChannelEventListener { + public void onChannelConnect(final String remoteAddr, final Channel channel); + + + public void onChannelClose(final String remoteAddr, final Channel channel); + + + public void onChannelException(final String remoteAddr, final Channel channel); + + + public void onChannelIdle(final String remoteAddr, final Channel channel); +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/CommandCustomHeader.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/CommandCustomHeader.java new file mode 100644 index 000000000..27b7f489a --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/CommandCustomHeader.java @@ -0,0 +1,29 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.remoting; + +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * RemotingCommand中自定义字段反射对象的公共接口 + * + * @author shijia.wxr + * @since 2013-7-13 + */ +public interface CommandCustomHeader { + public void checkFields() throws RemotingCommandException; +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/InvokeCallback.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/InvokeCallback.java new file mode 100644 index 000000000..e0d67ebb2 --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/InvokeCallback.java @@ -0,0 +1,29 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.remoting; + +import com.alibaba.rocketmq.remoting.netty.ResponseFuture; + + +/** + * 异步调用应答回调接口 + * + * @author shijia.wxr + * @since 2013-7-13 + */ +public interface InvokeCallback { + public void operationComplete(final ResponseFuture responseFuture); +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/RPCHook.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/RPCHook.java new file mode 100644 index 000000000..1ed3037b8 --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/RPCHook.java @@ -0,0 +1,11 @@ +package com.alibaba.rocketmq.remoting; + +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; + + +public interface RPCHook { + public void doBeforeRequest(final String remoteAddr, final RemotingCommand request); + + + public void doAfterResponse(final RemotingCommand request, final RemotingCommand response); +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/RemotingClient.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/RemotingClient.java new file mode 100644 index 000000000..4be8cbf74 --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/RemotingClient.java @@ -0,0 +1,63 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.remoting; + +import java.util.List; +import java.util.concurrent.ExecutorService; + +import com.alibaba.rocketmq.remoting.exception.RemotingConnectException; +import com.alibaba.rocketmq.remoting.exception.RemotingSendRequestException; +import com.alibaba.rocketmq.remoting.exception.RemotingTimeoutException; +import com.alibaba.rocketmq.remoting.exception.RemotingTooMuchRequestException; +import com.alibaba.rocketmq.remoting.netty.NettyRequestProcessor; +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; + + +/** + * 远程通信，Client接口 + * + * @author shijia.wxr + * @since 2013-7-13 + */ +public interface RemotingClient extends RemotingService { + + public void updateNameServerAddressList(final List addrs); + + + public List getNameServerAddressList(); + + + public RemotingCommand invokeSync(final String addr, final RemotingCommand request, + final long timeoutMillis) throws InterruptedException, RemotingConnectException, + RemotingSendRequestException, RemotingTimeoutException; + + + public void invokeAsync(final String addr, final RemotingCommand request, final long timeoutMillis, + final InvokeCallback invokeCallback) throws InterruptedException, RemotingConnectException, + RemotingTooMuchRequestException, RemotingTimeoutException, RemotingSendRequestException; + + + public void invokeOneway(final String addr, final RemotingCommand request, final long timeoutMillis) + throws InterruptedException, RemotingConnectException, RemotingTooMuchRequestException, + RemotingTimeoutException, RemotingSendRequestException; + + + public void registerProcessor(final int requestCode, final NettyRequestProcessor processor, + final ExecutorService executor); + + + public boolean isChannelWriteable(final String addr); +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/RemotingServer.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/RemotingServer.java new file mode 100644 index 000000000..891e3f789 --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/RemotingServer.java @@ -0,0 +1,73 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.remoting; + +import io.netty.channel.Channel; + +import java.util.concurrent.ExecutorService; + +import com.alibaba.rocketmq.remoting.exception.RemotingSendRequestException; +import com.alibaba.rocketmq.remoting.exception.RemotingTimeoutException; +import com.alibaba.rocketmq.remoting.exception.RemotingTooMuchRequestException; +import com.alibaba.rocketmq.remoting.netty.NettyRequestProcessor; +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; + + +/** + * 远程通信，Server接口 + * + * @author shijia.wxr + * @since 2013-7-13 + */ +public interface RemotingServer extends RemotingService { + + /** + * 注册请求处理器，ExecutorService必须要对应一个队列大小有限制的阻塞队列，防止OOM + * + * @param requestCode + * @param processor + * @param executor + */ + public void registerProcessor(final int requestCode, final NettyRequestProcessor processor, + final ExecutorService executor); + + + public void registerDefaultProcessor(final NettyRequestProcessor processor, final ExecutorService executor); + + + /** + * 服务器绑定的本地端口 + * + * @return PORT + */ + public int localListenPort(); + + + public RemotingCommand invokeSync(final Channel channel, final RemotingCommand request, + final long timeoutMillis) throws InterruptedException, RemotingSendRequestException, + RemotingTimeoutException; + + + public void invokeAsync(final Channel channel, final RemotingCommand request, final long timeoutMillis, + final InvokeCallback invokeCallback) throws InterruptedException, + RemotingTooMuchRequestException, RemotingTimeoutException, RemotingSendRequestException; + + + public void invokeOneway(final Channel channel, final RemotingCommand request, final long timeoutMillis) + throws InterruptedException, RemotingTooMuchRequestException, RemotingTimeoutException, + RemotingSendRequestException; + +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/RemotingService.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/RemotingService.java new file mode 100644 index 000000000..ae7fe223b --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/RemotingService.java @@ -0,0 +1,11 @@ +package com.alibaba.rocketmq.remoting; + +public interface RemotingService { + public void start(); + + + public void shutdown(); + + + public void registerRPCHook(RPCHook rpcHook); +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/annotation/CFNotNull.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/annotation/CFNotNull.java new file mode 100644 index 000000000..212af29e6 --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/annotation/CFNotNull.java @@ -0,0 +1,35 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.remoting.annotation; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + + +/** + * 表示字段不允许为空 + * + * @author shijia.wxr + * @since 2013-7-13 + */ +@Documented +@Retention(RetentionPolicy.RUNTIME) +@Target({ ElementType.FIELD, ElementType.METHOD, ElementType.PARAMETER, ElementType.LOCAL_VARIABLE }) +public @interface CFNotNull { +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/annotation/CFNullable.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/annotation/CFNullable.java new file mode 100644 index 000000000..caf209909 --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/annotation/CFNullable.java @@ -0,0 +1,35 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.remoting.annotation; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + + +/** + * 标识字段可以非空 + * + * @author shijia.wxr + * @since 2013-7-13 + */ +@Documented +@Retention(RetentionPolicy.RUNTIME) +@Target({ ElementType.FIELD, ElementType.METHOD, ElementType.PARAMETER, ElementType.LOCAL_VARIABLE }) +public @interface CFNullable { +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/common/Pair.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/common/Pair.java new file mode 100644 index 000000000..e3560f1aa --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/common/Pair.java @@ -0,0 +1,53 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.remoting.common; + +/** + * 包装2个对象 + * + * @author shijia.wxr + * @since 2013-7-13 + */ +public class Pair { + private T1 object1; + private T2 object2; + + + public Pair(T1 object1, T2 object2) { + this.object1 = object1; + this.object2 = object2; + } + + + public T1 getObject1() { + return object1; + } + + + public void setObject1(T1 object1) { + this.object1 = object1; + } + + + public T2 getObject2() { + return object2; + } + + + public void setObject2(T2 object2) { + this.object2 = object2; + } +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/common/RemotingHelper.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/common/RemotingHelper.java new file mode 100644 index 000000000..440b22761 --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/common/RemotingHelper.java @@ -0,0 +1,236 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.remoting.common; + +import io.netty.channel.Channel; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.nio.ByteBuffer; +import java.nio.channels.SocketChannel; + +import com.alibaba.rocketmq.remoting.exception.RemotingConnectException; +import com.alibaba.rocketmq.remoting.exception.RemotingSendRequestException; +import com.alibaba.rocketmq.remoting.exception.RemotingTimeoutException; +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; + + +/** + * 通信层一些辅助方法 + * + * @author shijia.wxr + * @since 2013-7-13 + */ +public class RemotingHelper { + public static final String RemotingLogName = "RocketmqRemoting"; + + + public static String exceptionSimpleDesc(final Throwable e) { + StringBuffer sb = new StringBuffer(); + if (e != null) { + sb.append(e.toString()); + + StackTraceElement[] stackTrace = e.getStackTrace(); + if (stackTrace != null && stackTrace.length > 0) { + StackTraceElement elment = stackTrace[0]; + sb.append(", "); + sb.append(elment.toString()); + } + } + + return sb.toString(); + } + + + /** + * IP:PORT + */ + public static SocketAddress string2SocketAddress(final String addr) { + String[] s = addr.split(":"); + InetSocketAddress isa = new InetSocketAddress(s[0], Integer.valueOf(s[1])); + return isa; + } + + + /** + * 短连接调用 TODO + */ + public static RemotingCommand invokeSync(final String addr, final RemotingCommand request, + final long timeoutMillis) throws InterruptedException, RemotingConnectException, + RemotingSendRequestException, RemotingTimeoutException { + long beginTime = System.currentTimeMillis(); + SocketAddress socketAddress = RemotingUtil.string2SocketAddress(addr); + SocketChannel socketChannel = RemotingUtil.connect(socketAddress); + if (socketChannel != null) { + boolean sendRequestOK = false; + + try { + // 使用阻塞模式 + socketChannel.configureBlocking(true); + /* + * FIXME The read methods in SocketChannel (and DatagramChannel) + * do notsupport timeouts + * http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=4614802 + */ + socketChannel.socket().setSoTimeout((int) timeoutMillis); + + // 发送数据 + ByteBuffer byteBufferRequest = request.encode(); + while (byteBufferRequest.hasRemaining()) { + int length = socketChannel.write(byteBufferRequest); + if (length > 0) { + if (byteBufferRequest.hasRemaining()) { + if ((System.currentTimeMillis() - beginTime) > timeoutMillis) { + // 发送请求超时 + throw new RemotingSendRequestException(addr); + } + } + } + else { + throw new RemotingSendRequestException(addr); + } + + // 比较土 + Thread.sleep(1); + } + + sendRequestOK = true; + + // 接收应答 SIZE + ByteBuffer byteBufferSize = ByteBuffer.allocate(4); + while (byteBufferSize.hasRemaining()) { + int length = socketChannel.read(byteBufferSize); + if (length > 0) { + if (byteBufferSize.hasRemaining()) { + if ((System.currentTimeMillis() - beginTime) > timeoutMillis) { + // 接收应答超时 + throw new RemotingTimeoutException(addr, timeoutMillis); + } + } + } + else { + throw new RemotingTimeoutException(addr, timeoutMillis); + } + + // 比较土 + Thread.sleep(1); + } + + // 接收应答 BODY + int size = byteBufferSize.getInt(0); + ByteBuffer byteBufferBody = ByteBuffer.allocate(size); + while (byteBufferBody.hasRemaining()) { + int length = socketChannel.read(byteBufferBody); + if (length > 0) { + if (byteBufferBody.hasRemaining()) { + if ((System.currentTimeMillis() - beginTime) > timeoutMillis) { + // 接收应答超时 + throw new RemotingTimeoutException(addr, timeoutMillis); + } + } + } + else { + throw new RemotingTimeoutException(addr, timeoutMillis); + } + + // 比较土 + Thread.sleep(1); + } + + // 对应答数据解码 + byteBufferBody.flip(); + return RemotingCommand.decode(byteBufferBody); + } + catch (IOException e) { + e.printStackTrace(); + + if (sendRequestOK) { + throw new RemotingTimeoutException(addr, timeoutMillis); + } + else { + throw new RemotingSendRequestException(addr); + } + } + finally { + try { + socketChannel.close(); + } + catch (IOException e) { + e.printStackTrace(); + } + } + } + else { + throw new RemotingConnectException(addr); + } + } + + + public static String parseChannelRemoteAddr(final Channel channel) { + if (null == channel) { + return ""; + } + final SocketAddress remote = channel.remoteAddress(); + final String addr = remote != null ? remote.toString() : ""; + + if (addr.length() > 0) { + int index = addr.lastIndexOf("/"); + if (index >= 0) { + return addr.substring(index + 1); + } + + return addr; + } + + return ""; + } + + + public static String parseChannelRemoteName(final Channel channel) { + if (null == channel) { + return ""; + } + final InetSocketAddress remote = (InetSocketAddress) channel.remoteAddress(); + if (remote != null) { + return remote.getAddress().getHostName(); + } + return ""; + } + + + public static String parseSocketAddressAddr(SocketAddress socketAddress) { + if (socketAddress != null) { + final String addr = socketAddress.toString(); + + if (addr.length() > 0) { + return addr.substring(1); + } + } + return ""; + } + + + public static String parseSocketAddressName(SocketAddress socketAddress) { + + final InetSocketAddress addrs = (InetSocketAddress) socketAddress; + if (addrs != null) { + return addrs.getAddress().getHostName(); + } + return ""; + } + +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/common/RemotingUtil.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/common/RemotingUtil.java new file mode 100644 index 000000000..f0fc054c8 --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/common/RemotingUtil.java @@ -0,0 +1,237 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.remoting.common; + +import io.netty.channel.Channel; +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelFutureListener; + +import java.io.IOException; +import java.lang.reflect.Method; +import java.net.Inet6Address; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.NetworkInterface; +import java.net.SocketAddress; +import java.net.SocketException; +import java.net.UnknownHostException; +import java.nio.channels.Selector; +import java.nio.channels.SocketChannel; +import java.nio.channels.spi.SelectorProvider; +import java.util.ArrayList; +import java.util.Enumeration; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +/** + * 网络相关方法 + * + * @author shijia.wxr + * @since 2013-7-13 + */ +public class RemotingUtil { + private static final Logger log = LoggerFactory.getLogger(RemotingHelper.RemotingLogName); + public static final String OS_NAME = System.getProperty("os.name"); + + private static boolean isLinuxPlatform = false; + private static boolean isWindowsPlatform = false; + + static { + if (OS_NAME != null && OS_NAME.toLowerCase().indexOf("linux") >= 0) { + isLinuxPlatform = true; + } + + if (OS_NAME != null && OS_NAME.toLowerCase().indexOf("windows") >= 0) { + isWindowsPlatform = true; + } + } + + + public static boolean isLinuxPlatform() { + return isLinuxPlatform; + } + + + public static boolean isWindowsPlatform() { + return isWindowsPlatform; + } + + + public static Selector openSelector() throws IOException { + Selector result = null; + // 在linux平台，尽量启用epoll实现 + if (isLinuxPlatform()) { + try { + final Class providerClazz = Class.forName("sun.nio.ch.EPollSelectorProvider"); + if (providerClazz != null) { + try { + final Method method = providerClazz.getMethod("provider"); + if (method != null) { + final SelectorProvider selectorProvider = (SelectorProvider) method.invoke(null); + if (selectorProvider != null) { + result = selectorProvider.openSelector(); + } + } + } + catch (final Exception e) { + // ignore + } + } + } + catch (final Exception e) { + // ignore + } + } + + if (result == null) { + result = Selector.open(); + } + + return result; + } + + + public static String getLocalAddress() { + try { + // 遍历网卡，查找一个非回路ip地址并返回 + Enumeration enumeration = NetworkInterface.getNetworkInterfaces(); + ArrayList ipv4Result = new ArrayList(); + ArrayList ipv6Result = new ArrayList(); + while (enumeration.hasMoreElements()) { + final NetworkInterface networkInterface = enumeration.nextElement(); + final Enumeration en = networkInterface.getInetAddresses(); + while (en.hasMoreElements()) { + final InetAddress address = en.nextElement(); + if (!address.isLoopbackAddress()) { + if (address instanceof Inet6Address) { + ipv6Result.add(normalizeHostAddress(address)); + } + else { + ipv4Result.add(normalizeHostAddress(address)); + } + } + } + } + + // 优先使用ipv4 + if (!ipv4Result.isEmpty()) { + for (String ip : ipv4Result) { + if (ip.startsWith("127.0") || ip.startsWith("192.168")) { + continue; + } + + return ip; + } + + // 取最后一个 + return ipv4Result.get(ipv4Result.size() - 1); + } + // 然后使用ipv6 + else if (!ipv6Result.isEmpty()) { + return ipv6Result.get(0); + } + // 然后使用本地ip + final InetAddress localHost = InetAddress.getLocalHost(); + return normalizeHostAddress(localHost); + } + catch (SocketException e) { + e.printStackTrace(); + } + catch (UnknownHostException e) { + e.printStackTrace(); + } + + return null; + } + + + public static String normalizeHostAddress(final InetAddress localHost) { + if (localHost instanceof Inet6Address) { + return "[" + localHost.getHostAddress() + "]"; + } + else { + return localHost.getHostAddress(); + } + } + + + /** + * IP:PORT + */ + public static SocketAddress string2SocketAddress(final String addr) { + String[] s = addr.split(":"); + InetSocketAddress isa = new InetSocketAddress(s[0], Integer.valueOf(s[1])); + return isa; + } + + + public static String socketAddress2String(final SocketAddress addr) { + StringBuilder sb = new StringBuilder(); + InetSocketAddress inetSocketAddress = (InetSocketAddress) addr; + sb.append(inetSocketAddress.getAddress().getHostAddress()); + sb.append(":"); + sb.append(inetSocketAddress.getPort()); + return sb.toString(); + } + + + public static SocketChannel connect(SocketAddress remote) { + return connect(remote, 1000 * 5); + } + + + public static SocketChannel connect(SocketAddress remote, final int timeoutMillis) { + SocketChannel sc = null; + try { + sc = SocketChannel.open(); + sc.configureBlocking(true); + sc.socket().setSoLinger(false, -1); + sc.socket().setTcpNoDelay(true); + sc.socket().setReceiveBufferSize(1024 * 64); + sc.socket().setSendBufferSize(1024 * 64); + sc.socket().connect(remote, timeoutMillis); + sc.configureBlocking(false); + return sc; + } + catch (Exception e) { + if (sc != null) { + try { + sc.close(); + } + catch (IOException e1) { + e1.printStackTrace(); + } + } + } + + return null; + } + + + public static void closeChannel(Channel channel) { + final String addrRemote = RemotingHelper.parseChannelRemoteAddr(channel); + channel.close().addListener(new ChannelFutureListener() { + @Override + public void operationComplete(ChannelFuture future) throws Exception { + log.info("closeChannel: close the connection to remote address[{}] result: {}", addrRemote, + future.isSuccess()); + } + }); + } + +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/common/SemaphoreReleaseOnlyOnce.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/common/SemaphoreReleaseOnlyOnce.java new file mode 100644 index 000000000..72230c972 --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/common/SemaphoreReleaseOnlyOnce.java @@ -0,0 +1,50 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.remoting.common; + +import java.util.concurrent.Semaphore; +import java.util.concurrent.atomic.AtomicBoolean; + + +/** + * 使用布尔原子变量，信号量保证只释放一次 + * + * @author shijia.wxr + * @since 2013-7-13 + */ +public class SemaphoreReleaseOnlyOnce { + private final AtomicBoolean released = new AtomicBoolean(false); + private final Semaphore semaphore; + + + public SemaphoreReleaseOnlyOnce(Semaphore semaphore) { + this.semaphore = semaphore; + } + + + public void release() { + if (this.semaphore != null) { + if (this.released.compareAndSet(false, true)) { + this.semaphore.release(); + } + } + } + + + public Semaphore getSemaphore() { + return semaphore; + } +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/common/ServiceThread.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/common/ServiceThread.java new file mode 100644 index 000000000..acc1982ca --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/common/ServiceThread.java @@ -0,0 +1,156 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.remoting.common; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +/** + * 后台服务线程基类 + * + * @author shijia.wxr + * @since 2013-7-13 + */ +public abstract class ServiceThread implements Runnable { + private static final Logger stlog = LoggerFactory.getLogger(RemotingHelper.RemotingLogName); + // 执行线程 + protected final Thread thread; + // 线程回收时间，默认90S + private static final long JoinTime = 90 * 1000; + // 是否已经被Notify过 + protected volatile boolean hasNotified = false; + // 线程是否已经停止 + protected volatile boolean stoped = false; + + + public ServiceThread() { + this.thread = new Thread(this, this.getServiceName()); + } + + + public abstract String getServiceName(); + + + public void start() { + this.thread.start(); + } + + + public void shutdown() { + this.shutdown(false); + } + + + public void stop() { + this.stop(false); + } + + + public void makeStop() { + this.stoped = true; + stlog.info("makestop thread " + this.getServiceName()); + } + + + public void stop(final boolean interrupt) { + this.stoped = true; + stlog.info("stop thread " + this.getServiceName() + " interrupt " + interrupt); + synchronized (this) { + if (!this.hasNotified) { + this.hasNotified = true; + this.notify(); + } + } + + if (interrupt) { + this.thread.interrupt(); + } + } + + + public void shutdown(final boolean interrupt) { + this.stoped = true; + stlog.info("shutdown thread " + this.getServiceName() + " interrupt " + interrupt); + synchronized (this) { + if (!this.hasNotified) { + this.hasNotified = true; + this.notify(); + } + } + + try { + if (interrupt) { + this.thread.interrupt(); + } + + long beginTime = System.currentTimeMillis(); + this.thread.join(this.getJointime()); + long eclipseTime = System.currentTimeMillis() - beginTime; + stlog.info("join thread " + this.getServiceName() + " eclipse time(ms) " + eclipseTime + " " + + this.getJointime()); + } + catch (InterruptedException e) { + e.printStackTrace(); + } + } + + + public void wakeup() { + synchronized (this) { + if (!this.hasNotified) { + this.hasNotified = true; + this.notify(); + } + } + } + + + protected void waitForRunning(long interval) { + synchronized (this) { + if (this.hasNotified) { + this.hasNotified = false; + this.onWaitEnd(); + return; + } + + try { + this.wait(interval); + } + catch (InterruptedException e) { + e.printStackTrace(); + } + finally { + this.hasNotified = false; + this.onWaitEnd(); + } + } + } + + + protected void onWaitEnd() { + } + + + public boolean isStoped() { + return stoped; + } + + + public long getJointime() { + return JoinTime; + } +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/exception/RemotingCommandException.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/exception/RemotingCommandException.java new file mode 100644 index 000000000..532320e22 --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/exception/RemotingCommandException.java @@ -0,0 +1,36 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.remoting.exception; + +/** + * 命令解析自定义字段时，校验字段有效性抛出异常 + * + * @author shijia.wxr + * @since 2013-7-13 + */ +public class RemotingCommandException extends RemotingException { + private static final long serialVersionUID = -6061365915274953096L; + + + public RemotingCommandException(String message) { + super(message, null); + } + + + public RemotingCommandException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/exception/RemotingConnectException.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/exception/RemotingConnectException.java new file mode 100644 index 000000000..4a2d88e15 --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/exception/RemotingConnectException.java @@ -0,0 +1,36 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.remoting.exception; + +/** + * Client连接Server失败，抛出此异常 + * + * @author shijia.wxr + * @since 2013-7-13 + */ +public class RemotingConnectException extends RemotingException { + private static final long serialVersionUID = -5565366231695911316L; + + + public RemotingConnectException(String addr) { + this(addr, null); + } + + + public RemotingConnectException(String addr, Throwable cause) { + super("connect to <" + addr + "> failed", cause); + } +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/exception/RemotingException.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/exception/RemotingException.java new file mode 100644 index 000000000..27e278f28 --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/exception/RemotingException.java @@ -0,0 +1,36 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.remoting.exception; + +/** + * 通信层异常父类 + * + * @author shijia.wxr + * @since 2013-7-13 + */ +public class RemotingException extends Exception { + private static final long serialVersionUID = -5690687334570505110L; + + + public RemotingException(String message) { + super(message); + } + + + public RemotingException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/exception/RemotingSendRequestException.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/exception/RemotingSendRequestException.java new file mode 100644 index 000000000..fc88c185e --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/exception/RemotingSendRequestException.java @@ -0,0 +1,36 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.remoting.exception; + +/** + * RPC调用中，客户端发送请求失败，抛出此异常 + * + * @author shijia.wxr + * @since 2013-7-13 + */ +public class RemotingSendRequestException extends RemotingException { + private static final long serialVersionUID = 5391285827332471674L; + + + public RemotingSendRequestException(String addr) { + this(addr, null); + } + + + public RemotingSendRequestException(String addr, Throwable cause) { + super("send request to <" + addr + "> failed", cause); + } +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/exception/RemotingTimeoutException.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/exception/RemotingTimeoutException.java new file mode 100644 index 000000000..1211c2c6e --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/exception/RemotingTimeoutException.java @@ -0,0 +1,42 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.remoting.exception; + +/** + * RPC调用超时异常 + * + * @author shijia.wxr + * @since 2013-7-13 + */ +public class RemotingTimeoutException extends RemotingException { + + private static final long serialVersionUID = 4106899185095245979L; + + + public RemotingTimeoutException(String message) { + super(message); + } + + + public RemotingTimeoutException(String addr, long timeoutMillis) { + this(addr, timeoutMillis, null); + } + + + public RemotingTimeoutException(String addr, long timeoutMillis, Throwable cause) { + super("wait response on the channel <" + addr + "> timeout, " + timeoutMillis + "(ms)", cause); + } +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/exception/RemotingTooMuchRequestException.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/exception/RemotingTooMuchRequestException.java new file mode 100644 index 000000000..499503e59 --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/exception/RemotingTooMuchRequestException.java @@ -0,0 +1,31 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.remoting.exception; + +/** + * 异步调用或者Oneway调用，堆积的请求超过信号量最大值 + * + * @author shijia.wxr + * @since 2013-7-13 + */ +public class RemotingTooMuchRequestException extends RemotingException { + private static final long serialVersionUID = 4326919581254519654L; + + + public RemotingTooMuchRequestException(String message) { + super(message); + } +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettyClientConfig.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettyClientConfig.java new file mode 100644 index 000000000..fb3fc9c92 --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettyClientConfig.java @@ -0,0 +1,139 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.remoting.netty; + +/** + * Netty客户端配置类 + * + * @author shijia.wxr + * @since 2013-7-13 + */ +public class NettyClientConfig { + // 处理Server Response/Request + private int clientWorkerThreads = 4; + private int clientCallbackExecutorThreads = Runtime.getRuntime().availableProcessors(); + private int clientOnewaySemaphoreValue = NettySystemConfig.ClientOnewaySemaphoreValue; + private int clientAsyncSemaphoreValue = NettySystemConfig.ClientAsyncSemaphoreValue; + private long connectTimeoutMillis = 3000; + // channel超过1分钟不被访问 就关闭 + private long channelNotActiveInterval = 1000 * 60; + + private int clientChannelMaxIdleTimeSeconds = 120; + + private int clientSocketSndBufSize = NettySystemConfig.SocketSndbufSize; + private int clientSocketRcvBufSize = NettySystemConfig.SocketRcvbufSize; + private boolean clientPooledByteBufAllocatorEnable = false; + + + public int getClientWorkerThreads() { + return clientWorkerThreads; + } + + + public void setClientWorkerThreads(int clientWorkerThreads) { + this.clientWorkerThreads = clientWorkerThreads; + } + + + public int getClientOnewaySemaphoreValue() { + return clientOnewaySemaphoreValue; + } + + + public void setClientOnewaySemaphoreValue(int clientOnewaySemaphoreValue) { + this.clientOnewaySemaphoreValue = clientOnewaySemaphoreValue; + } + + + public long getConnectTimeoutMillis() { + return connectTimeoutMillis; + } + + + public void setConnectTimeoutMillis(long connectTimeoutMillis) { + this.connectTimeoutMillis = connectTimeoutMillis; + } + + + public int getClientCallbackExecutorThreads() { + return clientCallbackExecutorThreads; + } + + + public void setClientCallbackExecutorThreads(int clientCallbackExecutorThreads) { + this.clientCallbackExecutorThreads = clientCallbackExecutorThreads; + } + + + public long getChannelNotActiveInterval() { + return channelNotActiveInterval; + } + + + public void setChannelNotActiveInterval(long channelNotActiveInterval) { + this.channelNotActiveInterval = channelNotActiveInterval; + } + + + public int getClientAsyncSemaphoreValue() { + return clientAsyncSemaphoreValue; + } + + + public void setClientAsyncSemaphoreValue(int clientAsyncSemaphoreValue) { + this.clientAsyncSemaphoreValue = clientAsyncSemaphoreValue; + } + + + public int getClientChannelMaxIdleTimeSeconds() { + return clientChannelMaxIdleTimeSeconds; + } + + + public void setClientChannelMaxIdleTimeSeconds(int clientChannelMaxIdleTimeSeconds) { + this.clientChannelMaxIdleTimeSeconds = clientChannelMaxIdleTimeSeconds; + } + + + public int getClientSocketSndBufSize() { + return clientSocketSndBufSize; + } + + + public void setClientSocketSndBufSize(int clientSocketSndBufSize) { + this.clientSocketSndBufSize = clientSocketSndBufSize; + } + + + public int getClientSocketRcvBufSize() { + return clientSocketRcvBufSize; + } + + + public void setClientSocketRcvBufSize(int clientSocketRcvBufSize) { + this.clientSocketRcvBufSize = clientSocketRcvBufSize; + } + + + public boolean isClientPooledByteBufAllocatorEnable() { + return clientPooledByteBufAllocatorEnable; + } + + + public void setClientPooledByteBufAllocatorEnable(boolean clientPooledByteBufAllocatorEnable) { + this.clientPooledByteBufAllocatorEnable = clientPooledByteBufAllocatorEnable; + } +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettyDecoder.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettyDecoder.java new file mode 100644 index 000000000..30e58a1bc --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettyDecoder.java @@ -0,0 +1,75 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.remoting.netty; + +import io.netty.buffer.ByteBuf; +import io.netty.channel.ChannelHandlerContext; +import io.netty.handler.codec.LengthFieldBasedFrameDecoder; + +import java.nio.ByteBuffer; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.rocketmq.remoting.common.RemotingHelper; +import com.alibaba.rocketmq.remoting.common.RemotingUtil; +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; + + +/** + * 协议解码器 + * + * @author shijia.wxr + * @since 2013-7-13 + */ +public class NettyDecoder extends LengthFieldBasedFrameDecoder { + private static final Logger log = LoggerFactory.getLogger(RemotingHelper.RemotingLogName); + private static final int FRAME_MAX_LENGTH = // + Integer.parseInt(System.getProperty("com.rocketmq.remoting.frameMaxLength", "8388608")); + + + public NettyDecoder() { + super(FRAME_MAX_LENGTH, 0, 4, 0, 4); + } + + + @Override + public Object decode(ChannelHandlerContext ctx, ByteBuf in) throws Exception { + ByteBuf frame = null; + try { + frame = (ByteBuf) super.decode(ctx, in); + if (null == frame) { + return null; + } + + ByteBuffer byteBuffer = frame.nioBuffer(); + + return RemotingCommand.decode(byteBuffer); + } + catch (Exception e) { + log.error("decode exception, " + RemotingHelper.parseChannelRemoteAddr(ctx.channel()), e); + // 这里关闭后， 会在pipeline中产生事件，通过具体的close事件来清理数据结构 + RemotingUtil.closeChannel(ctx.channel()); + } + finally { + if (null != frame) { + frame.release(); + } + } + + return null; + } +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettyEncoder.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettyEncoder.java new file mode 100644 index 000000000..357437e33 --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettyEncoder.java @@ -0,0 +1,62 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.remoting.netty; + +import io.netty.buffer.ByteBuf; +import io.netty.channel.ChannelHandlerContext; +import io.netty.handler.codec.MessageToByteEncoder; + +import java.nio.ByteBuffer; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.rocketmq.remoting.common.RemotingHelper; +import com.alibaba.rocketmq.remoting.common.RemotingUtil; +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; + + +/** + * 协议编码器 + * + * @author shijia.wxr + * @since 2013-7-13 + */ +public class NettyEncoder extends MessageToByteEncoder { + private static final Logger log = LoggerFactory.getLogger(RemotingHelper.RemotingLogName); + + + @Override + public void encode(ChannelHandlerContext ctx, RemotingCommand remotingCommand, ByteBuf out) + throws Exception { + try { + ByteBuffer header = remotingCommand.encodeHeader(); + out.writeBytes(header); + byte[] body = remotingCommand.getBody(); + if (body != null) { + out.writeBytes(body); + } + } + catch (Exception e) { + log.error("encode exception, " + RemotingHelper.parseChannelRemoteAddr(ctx.channel()), e); + if (remotingCommand != null) { + log.error(remotingCommand.toString()); + } + // 这里关闭后， 会在pipeline中产生事件，通过具体的close事件来清理数据结构 + RemotingUtil.closeChannel(ctx.channel()); + } + } +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettyEvent.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettyEvent.java new file mode 100644 index 000000000..35fc7d794 --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettyEvent.java @@ -0,0 +1,59 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.remoting.netty; + +import io.netty.channel.Channel; + + +/** + * Netty产生的各种事件 + * + * @author shijia.wxr + * @since 2013-7-13 + */ +public class NettyEvent { + private final NettyEventType type; + private final String remoteAddr; + private final Channel channel; + + + public NettyEvent(NettyEventType type, String remoteAddr, Channel channel) { + this.type = type; + this.remoteAddr = remoteAddr; + this.channel = channel; + } + + + public NettyEventType getType() { + return type; + } + + + public String getRemoteAddr() { + return remoteAddr; + } + + + public Channel getChannel() { + return channel; + } + + + @Override + public String toString() { + return "NettyEvent [type=" + type + ", remoteAddr=" + remoteAddr + ", channel=" + channel + "]"; + } +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettyEventType.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettyEventType.java new file mode 100644 index 000000000..24cb80e81 --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettyEventType.java @@ -0,0 +1,29 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.remoting.netty; + +/** + * Netty产生的事件类型 + * + * @author shijia.wxr + * @since 2013-7-13 + */ +public enum NettyEventType { + CONNECT, + CLOSE, + IDLE, + EXCEPTION +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettyRemotingAbstract.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettyRemotingAbstract.java new file mode 100644 index 000000000..341ac4d95 --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettyRemotingAbstract.java @@ -0,0 +1,510 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.remoting.netty; + +import io.netty.channel.Channel; +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelFutureListener; +import io.netty.channel.ChannelHandlerContext; + +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map.Entry; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.rocketmq.remoting.ChannelEventListener; +import com.alibaba.rocketmq.remoting.InvokeCallback; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.remoting.common.Pair; +import com.alibaba.rocketmq.remoting.common.RemotingHelper; +import com.alibaba.rocketmq.remoting.common.SemaphoreReleaseOnlyOnce; +import com.alibaba.rocketmq.remoting.common.ServiceThread; +import com.alibaba.rocketmq.remoting.exception.RemotingSendRequestException; +import com.alibaba.rocketmq.remoting.exception.RemotingTimeoutException; +import com.alibaba.rocketmq.remoting.exception.RemotingTooMuchRequestException; +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; +import com.alibaba.rocketmq.remoting.protocol.RemotingSysResponseCode; + + +/** + * Server与Client公用抽象类 + * + * @author shijia.wxr + * @since 2013-7-13 + */ +public abstract class NettyRemotingAbstract { + private static final Logger plog = LoggerFactory.getLogger(RemotingHelper.RemotingLogName); + + // 信号量，Oneway情况会使用，防止本地Netty缓存请求过多 + protected final Semaphore semaphoreOneway; + + // 信号量，异步调用情况会使用，防止本地Netty缓存请求过多 + protected final Semaphore semaphoreAsync; + + // 缓存所有对外请求 + protected final ConcurrentHashMap responseTable = + new ConcurrentHashMap(256); + + // 默认请求代码处理器 + protected Pair defaultRequestProcessor; + + // 注册的各个RPC处理器 + protected final HashMap> processorTable = + new HashMap>(64); + + protected final NettyEventExecuter nettyEventExecuter = new NettyEventExecuter(); + + + public abstract ChannelEventListener getChannelEventListener(); + + + public abstract RPCHook getRPCHook(); + + + public void putNettyEvent(final NettyEvent event) { + this.nettyEventExecuter.putNettyEvent(event); + } + + class NettyEventExecuter extends ServiceThread { + private final LinkedBlockingQueue eventQueue = new LinkedBlockingQueue(); + private final int MaxSize = 10000; + + + public void putNettyEvent(final NettyEvent event) { + if (this.eventQueue.size() <= MaxSize) { + this.eventQueue.add(event); + } + else { + plog.warn("event queue size[{}] enough, so drop this event {}", this.eventQueue.size(), + event.toString()); + } + } + + + @Override + public void run() { + plog.info(this.getServiceName() + " service started"); + + final ChannelEventListener listener = NettyRemotingAbstract.this.getChannelEventListener(); + + while (!this.isStoped()) { + try { + NettyEvent event = this.eventQueue.poll(3000, TimeUnit.MILLISECONDS); + if (event != null && listener != null) { + switch (event.getType()) { + case IDLE: + listener.onChannelIdle(event.getRemoteAddr(), event.getChannel()); + break; + case CLOSE: + listener.onChannelClose(event.getRemoteAddr(), event.getChannel()); + break; + case CONNECT: + listener.onChannelConnect(event.getRemoteAddr(), event.getChannel()); + break; + case EXCEPTION: + listener.onChannelException(event.getRemoteAddr(), event.getChannel()); + break; + default: + break; + + } + } + } + catch (Exception e) { + plog.warn(this.getServiceName() + " service has exception. ", e); + } + } + + plog.info(this.getServiceName() + " service end"); + } + + + @Override + public String getServiceName() { + return NettyEventExecuter.class.getSimpleName(); + } + } + + + public NettyRemotingAbstract(final int permitsOneway, final int permitsAsync) { + this.semaphoreOneway = new Semaphore(permitsOneway, true); + this.semaphoreAsync = new Semaphore(permitsAsync, true); + } + + + public void processRequestCommand(final ChannelHandlerContext ctx, final RemotingCommand cmd) { + final Pair matched = this.processorTable.get(cmd.getCode()); + final Pair pair = + null == matched ? this.defaultRequestProcessor : matched; + + if (pair != null) { + Runnable run = new Runnable() { + @Override + public void run() { + try { + RPCHook rpcHook = NettyRemotingAbstract.this.getRPCHook(); + if (rpcHook != null) { + rpcHook + .doBeforeRequest(RemotingHelper.parseChannelRemoteAddr(ctx.channel()), cmd); + } + + final RemotingCommand response = pair.getObject1().processRequest(ctx, cmd); + if (rpcHook != null) { + rpcHook.doAfterResponse(cmd, response); + } + + // Oneway形式忽略应答结果 + if (!cmd.isOnewayRPC()) { + if (response != null) { + response.setOpaque(cmd.getOpaque()); + response.markResponseType(); + try { + ctx.writeAndFlush(response); + } + catch (Throwable e) { + plog.error("process request over, but response failed", e); + plog.error(cmd.toString()); + plog.error(response.toString()); + } + } + else { + // 收到请求，但是没有返回应答，可能是processRequest中进行了应答，忽略这种情况 + } + } + } + catch (Throwable e) { + plog.error("process request exception", e); + plog.error(cmd.toString()); + + if (!cmd.isOnewayRPC()) { + final RemotingCommand response = + RemotingCommand.createResponseCommand( + RemotingSysResponseCode.SYSTEM_ERROR,// + RemotingHelper.exceptionSimpleDesc(e)); + response.setOpaque(cmd.getOpaque()); + ctx.writeAndFlush(response); + } + } + } + }; + + try { + // 这里需要做流控，要求线程池对应的队列必须是有大小限制的 + pair.getObject2().submit(run); + } + catch (RejectedExecutionException e) { + // 每个线程10s打印一次 + if ((System.currentTimeMillis() % 10000) == 0) { + plog.warn(RemotingHelper.parseChannelRemoteAddr(ctx.channel()) // + + ", too many requests and system thread pool busy, RejectedExecutionException " // + + pair.getObject2().toString() // + + " request code: " + cmd.getCode()); + } + + if (!cmd.isOnewayRPC()) { + final RemotingCommand response = + RemotingCommand.createResponseCommand(RemotingSysResponseCode.SYSTEM_BUSY, + "too many requests and system thread pool busy, please try another server"); + response.setOpaque(cmd.getOpaque()); + ctx.writeAndFlush(response); + } + } + } + else { + String error = " request type " + cmd.getCode() + " not supported"; + final RemotingCommand response = + RemotingCommand.createResponseCommand(RemotingSysResponseCode.REQUEST_CODE_NOT_SUPPORTED, + error); + response.setOpaque(cmd.getOpaque()); + ctx.writeAndFlush(response); + plog.error(RemotingHelper.parseChannelRemoteAddr(ctx.channel()) + error); + } + } + + + public void processResponseCommand(ChannelHandlerContext ctx, RemotingCommand cmd) { + final ResponseFuture responseFuture = responseTable.get(cmd.getOpaque()); + if (responseFuture != null) { + responseFuture.setResponseCommand(cmd); + + responseFuture.release(); + + // 异步调用 + if (responseFuture.getInvokeCallback() != null) { + boolean runInThisThread = false; + ExecutorService executor = this.getCallbackExecutor(); + if (executor != null) { + try { + executor.submit(new Runnable() { + @Override + public void run() { + try { + responseFuture.executeInvokeCallback(); + } + catch (Throwable e) { + plog.warn("excute callback in executor exception, and callback throw", e); + } + } + }); + } + catch (Exception e) { + runInThisThread = true; + plog.warn("excute callback in executor exception, maybe executor busy", e); + } + } + else { + runInThisThread = true; + } + + if (runInThisThread) { + try { + responseFuture.executeInvokeCallback(); + } + catch (Throwable e) { + plog.warn("executeInvokeCallback Exception", e); + } + } + } + // 同步调用 + else { + responseFuture.putResponse(cmd); + } + } + else { + plog.warn("receive response, but not matched any request, " + + RemotingHelper.parseChannelRemoteAddr(ctx.channel())); + plog.warn(cmd.toString()); + } + + responseTable.remove(cmd.getOpaque()); + } + + + public void processMessageReceived(ChannelHandlerContext ctx, RemotingCommand msg) throws Exception { + final RemotingCommand cmd = msg; + if (cmd != null) { + switch (cmd.getType()) { + case REQUEST_COMMAND: + processRequestCommand(ctx, cmd); + break; + case RESPONSE_COMMAND: + processResponseCommand(ctx, cmd); + break; + default: + break; + } + } + } + + + abstract public ExecutorService getCallbackExecutor(); + + + public void scanResponseTable() { + Iterator> it = this.responseTable.entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + ResponseFuture rep = next.getValue(); + + if ((rep.getBeginTimestamp() + rep.getTimeoutMillis() + 1000) <= System.currentTimeMillis()) { + it.remove(); + try { + rep.executeInvokeCallback(); + } + catch (Throwable e) { + plog.warn("scanResponseTable, operationComplete Exception", e); + } + finally { + rep.release(); + } + + plog.warn("remove timeout request, " + rep); + } + } + } + + + public RemotingCommand invokeSyncImpl(final Channel channel, final RemotingCommand request, + final long timeoutMillis) throws InterruptedException, RemotingSendRequestException, + RemotingTimeoutException { + try { + final ResponseFuture responseFuture = + new ResponseFuture(request.getOpaque(), timeoutMillis, null, null); + this.responseTable.put(request.getOpaque(), responseFuture); + channel.writeAndFlush(request).addListener(new ChannelFutureListener() { + @Override + public void operationComplete(ChannelFuture f) throws Exception { + if (f.isSuccess()) { + responseFuture.setSendRequestOK(true); + return; + } + else { + responseFuture.setSendRequestOK(false); + } + + responseTable.remove(request.getOpaque()); + responseFuture.setCause(f.cause()); + responseFuture.putResponse(null); + plog.warn("send a request command to channel <" + channel.remoteAddress() + "> failed."); + plog.warn(request.toString()); + } + }); + + RemotingCommand responseCommand = responseFuture.waitResponse(timeoutMillis); + if (null == responseCommand) { + // 发送请求成功，读取应答超时 + if (responseFuture.isSendRequestOK()) { + throw new RemotingTimeoutException(RemotingHelper.parseChannelRemoteAddr(channel), + timeoutMillis, responseFuture.getCause()); + } + // 发送请求失败 + else { + throw new RemotingSendRequestException(RemotingHelper.parseChannelRemoteAddr(channel), + responseFuture.getCause()); + } + } + + return responseCommand; + } + finally { + this.responseTable.remove(request.getOpaque()); + } + } + + + public void invokeAsyncImpl(final Channel channel, final RemotingCommand request, + final long timeoutMillis, final InvokeCallback invokeCallback) throws InterruptedException, + RemotingTooMuchRequestException, RemotingTimeoutException, RemotingSendRequestException { + boolean acquired = this.semaphoreAsync.tryAcquire(timeoutMillis, TimeUnit.MILLISECONDS); + if (acquired) { + final SemaphoreReleaseOnlyOnce once = new SemaphoreReleaseOnlyOnce(this.semaphoreAsync); + + final ResponseFuture responseFuture = + new ResponseFuture(request.getOpaque(), timeoutMillis, invokeCallback, once); + this.responseTable.put(request.getOpaque(), responseFuture); + try { + channel.writeAndFlush(request).addListener(new ChannelFutureListener() { + @Override + public void operationComplete(ChannelFuture f) throws Exception { + if (f.isSuccess()) { + responseFuture.setSendRequestOK(true); + return; + } + else { + responseFuture.setSendRequestOK(false); + } + + responseFuture.putResponse(null); + responseTable.remove(request.getOpaque()); + try { + responseFuture.executeInvokeCallback(); + } + catch (Throwable e) { + plog.warn("excute callback in writeAndFlush addListener, and callback throw", e); + } + finally { + responseFuture.release(); + } + + plog.warn("send a request command to channel <{}> failed.", + RemotingHelper.parseChannelRemoteAddr(channel)); + plog.warn(request.toString()); + } + }); + } + catch (Exception e) { + responseFuture.release(); + plog.warn( + "send a request command to channel <" + RemotingHelper.parseChannelRemoteAddr(channel) + + "> Exception", e); + throw new RemotingSendRequestException(RemotingHelper.parseChannelRemoteAddr(channel), e); + } + } + else { + if (timeoutMillis <= 0) { + throw new RemotingTooMuchRequestException("invokeAsyncImpl invoke too fast"); + } + else { + String info = + String + .format( + "invokeAsyncImpl tryAcquire semaphore timeout, %dms, waiting thread nums: %d semaphoreAsyncValue: %d", // + timeoutMillis,// + this.semaphoreAsync.getQueueLength(),// + this.semaphoreAsync.availablePermits()// + ); + plog.warn(info); + plog.warn(request.toString()); + throw new RemotingTimeoutException(info); + } + } + } + + + public void invokeOnewayImpl(final Channel channel, final RemotingCommand request, + final long timeoutMillis) throws InterruptedException, RemotingTooMuchRequestException, + RemotingTimeoutException, RemotingSendRequestException { + request.markOnewayRPC(); + boolean acquired = this.semaphoreOneway.tryAcquire(timeoutMillis, TimeUnit.MILLISECONDS); + if (acquired) { + final SemaphoreReleaseOnlyOnce once = new SemaphoreReleaseOnlyOnce(this.semaphoreOneway); + try { + channel.writeAndFlush(request).addListener(new ChannelFutureListener() { + @Override + public void operationComplete(ChannelFuture f) throws Exception { + once.release(); + if (!f.isSuccess()) { + plog.warn("send a request command to channel <" + channel.remoteAddress() + + "> failed."); + plog.warn(request.toString()); + } + } + }); + } + catch (Exception e) { + once.release(); + plog.warn("write send a request command to channel <" + channel.remoteAddress() + "> failed."); + throw new RemotingSendRequestException(RemotingHelper.parseChannelRemoteAddr(channel), e); + } + } + else { + if (timeoutMillis <= 0) { + throw new RemotingTooMuchRequestException("invokeOnewayImpl invoke too fast"); + } + else { + String info = + String + .format( + "invokeOnewayImpl tryAcquire semaphore timeout, %dms, waiting thread nums: %d semaphoreAsyncValue: %d", // + timeoutMillis,// + this.semaphoreAsync.getQueueLength(),// + this.semaphoreAsync.availablePermits()// + ); + plog.warn(info); + plog.warn(request.toString()); + throw new RemotingTimeoutException(info); + } + } + } +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettyRemotingClient.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettyRemotingClient.java new file mode 100644 index 000000000..82194e901 --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettyRemotingClient.java @@ -0,0 +1,776 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.remoting.netty; + +import io.netty.bootstrap.Bootstrap; +import io.netty.channel.Channel; +import io.netty.channel.ChannelDuplexHandler; +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInitializer; +import io.netty.channel.ChannelOption; +import io.netty.channel.ChannelPromise; +import io.netty.channel.EventLoopGroup; +import io.netty.channel.SimpleChannelInboundHandler; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.socket.SocketChannel; +import io.netty.channel.socket.nio.NioSocketChannel; +import io.netty.handler.timeout.IdleState; +import io.netty.handler.timeout.IdleStateEvent; +import io.netty.handler.timeout.IdleStateHandler; +import io.netty.util.concurrent.DefaultEventExecutorGroup; + +import java.net.SocketAddress; +import java.util.Collections; +import java.util.List; +import java.util.Random; +import java.util.Timer; +import java.util.TimerTask; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.rocketmq.remoting.ChannelEventListener; +import com.alibaba.rocketmq.remoting.InvokeCallback; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.remoting.RemotingClient; +import com.alibaba.rocketmq.remoting.common.Pair; +import com.alibaba.rocketmq.remoting.common.RemotingHelper; +import com.alibaba.rocketmq.remoting.common.RemotingUtil; +import com.alibaba.rocketmq.remoting.exception.RemotingConnectException; +import com.alibaba.rocketmq.remoting.exception.RemotingSendRequestException; +import com.alibaba.rocketmq.remoting.exception.RemotingTimeoutException; +import com.alibaba.rocketmq.remoting.exception.RemotingTooMuchRequestException; +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; + + +/** + * Remoting客户端实现 + * + * @author shijia.wxr + * @since 2013-7-13 + */ +public class NettyRemotingClient extends NettyRemotingAbstract implements RemotingClient { + private static final Logger log = LoggerFactory.getLogger(RemotingHelper.RemotingLogName); + + private static final long LockTimeoutMillis = 3000; + + private final NettyClientConfig nettyClientConfig; + private final Bootstrap bootstrap = new Bootstrap(); + private final EventLoopGroup eventLoopGroupWorker; + private DefaultEventExecutorGroup defaultEventExecutorGroup; + + private final Lock lockChannelTables = new ReentrantLock(); + private final ConcurrentHashMap channelTables = + new ConcurrentHashMap(); + + // 定时器 + private final Timer timer = new Timer("ClientHouseKeepingService", true); + + // Name server相关 + private final AtomicReference> namesrvAddrList = new AtomicReference>(); + private final AtomicReference namesrvAddrChoosed = new AtomicReference(); + private final AtomicInteger namesrvIndex = new AtomicInteger(initValueIndex()); + private final Lock lockNamesrvChannel = new ReentrantLock(); + + // 处理Callback应答器 + private final ExecutorService publicExecutor; + + private final ChannelEventListener channelEventListener; + + private RPCHook rpcHook; + + class ChannelWrapper { + private final ChannelFuture channelFuture; + + + public ChannelWrapper(ChannelFuture channelFuture) { + this.channelFuture = channelFuture; + } + + + public boolean isOK() { + return (this.channelFuture.channel() != null && this.channelFuture.channel().isActive()); + } + + + public boolean isWriteable() { + return this.channelFuture.channel().isWritable(); + } + + + private Channel getChannel() { + return this.channelFuture.channel(); + } + + + public ChannelFuture getChannelFuture() { + return channelFuture; + } + } + + class NettyClientHandler extends SimpleChannelInboundHandler { + + @Override + protected void channelRead0(ChannelHandlerContext ctx, RemotingCommand msg) throws Exception { + processMessageReceived(ctx, msg); + + } + } + + class NettyConnetManageHandler extends ChannelDuplexHandler { + @Override + public void connect(ChannelHandlerContext ctx, SocketAddress remoteAddress, + SocketAddress localAddress, ChannelPromise promise) throws Exception { + final String local = localAddress == null ? "UNKNOW" : localAddress.toString(); + final String remote = remoteAddress == null ? "UNKNOW" : remoteAddress.toString(); + log.info("NETTY CLIENT PIPELINE: CONNECT {} => {}", local, remote); + super.connect(ctx, remoteAddress, localAddress, promise); + + if (NettyRemotingClient.this.channelEventListener != null) { + NettyRemotingClient.this.putNettyEvent(new NettyEvent(NettyEventType.CONNECT, remoteAddress + .toString(), ctx.channel())); + } + } + + + @Override + public void disconnect(ChannelHandlerContext ctx, ChannelPromise promise) throws Exception { + final String remoteAddress = RemotingHelper.parseChannelRemoteAddr(ctx.channel()); + log.info("NETTY CLIENT PIPELINE: DISCONNECT {}", remoteAddress); + closeChannel(ctx.channel()); + super.disconnect(ctx, promise); + + if (NettyRemotingClient.this.channelEventListener != null) { + NettyRemotingClient.this.putNettyEvent(new NettyEvent(NettyEventType.CLOSE, remoteAddress + .toString(), ctx.channel())); + } + } + + + @Override + public void close(ChannelHandlerContext ctx, ChannelPromise promise) throws Exception { + final String remoteAddress = RemotingHelper.parseChannelRemoteAddr(ctx.channel()); + log.info("NETTY CLIENT PIPELINE: CLOSE {}", remoteAddress); + closeChannel(ctx.channel()); + super.close(ctx, promise); + + if (NettyRemotingClient.this.channelEventListener != null) { + NettyRemotingClient.this.putNettyEvent(new NettyEvent(NettyEventType.CLOSE, remoteAddress + .toString(), ctx.channel())); + } + } + + + @Override + public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { + final String remoteAddress = RemotingHelper.parseChannelRemoteAddr(ctx.channel()); + log.warn("NETTY CLIENT PIPELINE: exceptionCaught {}", remoteAddress); + log.warn("NETTY CLIENT PIPELINE: exceptionCaught exception.", cause); + closeChannel(ctx.channel()); + if (NettyRemotingClient.this.channelEventListener != null) { + NettyRemotingClient.this.putNettyEvent(new NettyEvent(NettyEventType.EXCEPTION, remoteAddress + .toString(), ctx.channel())); + } + } + + + @Override + public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception { + if (evt instanceof IdleStateEvent) { + IdleStateEvent evnet = (IdleStateEvent) evt; + if (evnet.state().equals(IdleState.ALL_IDLE)) { + final String remoteAddress = RemotingHelper.parseChannelRemoteAddr(ctx.channel()); + log.warn("NETTY CLIENT PIPELINE: IDLE exception [{}]", remoteAddress); + closeChannel(ctx.channel()); + if (NettyRemotingClient.this.channelEventListener != null) { + NettyRemotingClient.this.putNettyEvent(new NettyEvent(NettyEventType.IDLE, + remoteAddress.toString(), ctx.channel())); + } + } + } + + ctx.fireUserEventTriggered(evt); + } + } + + + private static int initValueIndex() { + Random r = new Random(); + + return Math.abs(r.nextInt() % 999) % 999; + } + + + public NettyRemotingClient(final NettyClientConfig nettyClientConfig) { + this(nettyClientConfig, null); + } + + + public NettyRemotingClient(final NettyClientConfig nettyClientConfig,// + final ChannelEventListener channelEventListener) { + super(nettyClientConfig.getClientOnewaySemaphoreValue(), nettyClientConfig + .getClientAsyncSemaphoreValue()); + this.nettyClientConfig = nettyClientConfig; + this.channelEventListener = channelEventListener; + + int publicThreadNums = nettyClientConfig.getClientCallbackExecutorThreads(); + if (publicThreadNums <= 0) { + publicThreadNums = 4; + } + + this.publicExecutor = Executors.newFixedThreadPool(publicThreadNums, new ThreadFactory() { + private AtomicInteger threadIndex = new AtomicInteger(0); + + + @Override + public Thread newThread(Runnable r) { + return new Thread(r, "NettyClientPublicExecutor_" + this.threadIndex.incrementAndGet()); + } + }); + + this.eventLoopGroupWorker = new NioEventLoopGroup(1, new ThreadFactory() { + private AtomicInteger threadIndex = new AtomicInteger(0); + + + @Override + public Thread newThread(Runnable r) { + return new Thread(r, String.format("NettyClientSelector_%d", + this.threadIndex.incrementAndGet())); + } + }); + } + + + @Override + public void start() { + this.defaultEventExecutorGroup = new DefaultEventExecutorGroup(// + nettyClientConfig.getClientWorkerThreads(), // + new ThreadFactory() { + + private AtomicInteger threadIndex = new AtomicInteger(0); + + + @Override + public Thread newThread(Runnable r) { + return new Thread(r, "NettyClientWorkerThread_" + this.threadIndex.incrementAndGet()); + } + }); + + Bootstrap handler = this.bootstrap.group(this.eventLoopGroupWorker).channel(NioSocketChannel.class)// + // + .option(ChannelOption.TCP_NODELAY, true) + // + .option(ChannelOption.SO_KEEPALIVE, false) + // + .option(ChannelOption.SO_SNDBUF, nettyClientConfig.getClientSocketSndBufSize()) + // + .option(ChannelOption.SO_RCVBUF, nettyClientConfig.getClientSocketRcvBufSize()) + // + .handler(new ChannelInitializer() { + @Override + public void initChannel(SocketChannel ch) throws Exception { + ch.pipeline().addLast(// + defaultEventExecutorGroup, // + new NettyEncoder(), // + new NettyDecoder(), // + new IdleStateHandler(0, 0, nettyClientConfig.getClientChannelMaxIdleTimeSeconds()),// + new NettyConnetManageHandler(), // + new NettyClientHandler()); + } + }); + + // 每隔1秒扫描下异步调用超时情况 + this.timer.scheduleAtFixedRate(new TimerTask() { + + @Override + public void run() { + try { + NettyRemotingClient.this.scanResponseTable(); + } + catch (Exception e) { + log.error("scanResponseTable exception", e); + } + } + }, 1000 * 3, 1000); + + if (this.channelEventListener != null) { + this.nettyEventExecuter.start(); + } + } + + + @Override + public void shutdown() { + try { + this.timer.cancel(); + + for (ChannelWrapper cw : this.channelTables.values()) { + this.closeChannel(null, cw.getChannel()); + } + + this.channelTables.clear(); + + this.eventLoopGroupWorker.shutdownGracefully(); + + if (this.nettyEventExecuter != null) { + this.nettyEventExecuter.shutdown(); + } + + if (this.defaultEventExecutorGroup != null) { + this.defaultEventExecutorGroup.shutdownGracefully(); + } + } + catch (Exception e) { + log.error("NettyRemotingClient shutdown exception, ", e); + } + + if (this.publicExecutor != null) { + try { + this.publicExecutor.shutdown(); + } + catch (Exception e) { + log.error("NettyRemotingServer shutdown exception, ", e); + } + } + } + + + private Channel getAndCreateChannel(final String addr) throws InterruptedException { + if (null == addr) + return getAndCreateNameserverChannel(); + + ChannelWrapper cw = this.channelTables.get(addr); + if (cw != null && cw.isOK()) { + return cw.getChannel(); + } + + return this.createChannel(addr); + } + + + private Channel getAndCreateNameserverChannel() throws InterruptedException { + String addr = this.namesrvAddrChoosed.get(); + if (addr != null) { + ChannelWrapper cw = this.channelTables.get(addr); + if (cw != null && cw.isOK()) { + return cw.getChannel(); + } + } + + final List addrList = this.namesrvAddrList.get(); + // 加锁，尝试创建连接 + if (this.lockNamesrvChannel.tryLock(LockTimeoutMillis, TimeUnit.MILLISECONDS)) { + try { + addr = this.namesrvAddrChoosed.get(); + if (addr != null) { + ChannelWrapper cw = this.channelTables.get(addr); + if (cw != null && cw.isOK()) { + return cw.getChannel(); + } + } + + if (addrList != null && !addrList.isEmpty()) { + for (int i = 0; i < addrList.size(); i++) { + int index = this.namesrvIndex.incrementAndGet(); + index = Math.abs(index); + index = index % addrList.size(); + String newAddr = addrList.get(index); + + this.namesrvAddrChoosed.set(newAddr); + Channel channelNew = this.createChannel(newAddr); + if (channelNew != null) + return channelNew; + } + } + } + catch (Exception e) { + log.error("getAndCreateNameserverChannel: create name server channel exception", e); + } + finally { + this.lockNamesrvChannel.unlock(); + } + } + else { + log.warn("getAndCreateNameserverChannel: try to lock name server, but timeout, {}ms", + LockTimeoutMillis); + } + + return null; + } + + + private Channel createChannel(final String addr) throws InterruptedException { + ChannelWrapper cw = this.channelTables.get(addr); + if (cw != null && cw.isOK()) { + return cw.getChannel(); + } + + // 进入临界区后，不能有阻塞操作，网络连接采用异步方式 + if (this.lockChannelTables.tryLock(LockTimeoutMillis, TimeUnit.MILLISECONDS)) { + try { + boolean createNewConnection = false; + cw = this.channelTables.get(addr); + if (cw != null) { + // channel正常 + if (cw.isOK()) { + return cw.getChannel(); + } + // 正在连接，退出锁等待 + else if (!cw.getChannelFuture().isDone()) { + createNewConnection = false; + } + // 说明连接不成功 + else { + this.channelTables.remove(addr); + createNewConnection = true; + } + } + // ChannelWrapper不存在 + else { + createNewConnection = true; + } + + if (createNewConnection) { + ChannelFuture channelFuture = + this.bootstrap.connect(RemotingHelper.string2SocketAddress(addr)); + log.info("createChannel: begin to connect remote host[{}] asynchronously", addr); + cw = new ChannelWrapper(channelFuture); + this.channelTables.put(addr, cw); + } + } + catch (Exception e) { + log.error("createChannel: create channel exception", e); + } + finally { + this.lockChannelTables.unlock(); + } + } + else { + log.warn("createChannel: try to lock channel table, but timeout, {}ms", LockTimeoutMillis); + } + + if (cw != null) { + ChannelFuture channelFuture = cw.getChannelFuture(); + if (channelFuture.awaitUninterruptibly(this.nettyClientConfig.getConnectTimeoutMillis())) { + if (cw.isOK()) { + log.info("createChannel: connect remote host[{}] success, {}", addr, + channelFuture.toString()); + return cw.getChannel(); + } + else { + log.warn( + "createChannel: connect remote host[" + addr + "] failed, " + + channelFuture.toString(), channelFuture.cause()); + } + } + else { + log.warn("createChannel: connect remote host[{}] timeout {}ms, {}", addr, + this.nettyClientConfig.getConnectTimeoutMillis(), channelFuture.toString()); + } + } + + return null; + } + + + public void closeChannel(final String addr, final Channel channel) { + if (null == channel) + return; + + final String addrRemote = null == addr ? RemotingHelper.parseChannelRemoteAddr(channel) : addr; + + try { + if (this.lockChannelTables.tryLock(LockTimeoutMillis, TimeUnit.MILLISECONDS)) { + try { + boolean removeItemFromTable = true; + final ChannelWrapper prevCW = this.channelTables.get(addrRemote); + + log.info("closeChannel: begin close the channel[{}] Found: {}", addrRemote, + (prevCW != null)); + + if (null == prevCW) { + log.info( + "closeChannel: the channel[{}] has been removed from the channel table before", + addrRemote); + removeItemFromTable = false; + } + else if (prevCW.getChannel() != channel) { + log.info( + "closeChannel: the channel[{}] has been closed before, and has been created again, nothing to do.", + addrRemote); + removeItemFromTable = false; + } + + if (removeItemFromTable) { + this.channelTables.remove(addrRemote); + log.info("closeChannel: the channel[{}] was removed from channel table", addrRemote); + } + + RemotingUtil.closeChannel(channel); + } + catch (Exception e) { + log.error("closeChannel: close the channel exception", e); + } + finally { + this.lockChannelTables.unlock(); + } + } + else { + log.warn("closeChannel: try to lock channel table, but timeout, {}ms", LockTimeoutMillis); + } + } + catch (InterruptedException e) { + log.error("closeChannel exception", e); + } + } + + + public void closeChannel(final Channel channel) { + if (null == channel) + return; + + try { + if (this.lockChannelTables.tryLock(LockTimeoutMillis, TimeUnit.MILLISECONDS)) { + try { + boolean removeItemFromTable = true; + ChannelWrapper prevCW = null; + String addrRemote = null; + for (String key : channelTables.keySet()) { + ChannelWrapper prev = this.channelTables.get(key); + if (prev.getChannel() != null) { + if (prev.getChannel() == channel) { + prevCW = prev; + addrRemote = key; + break; + } + } + } + + if (null == prevCW) { + log.info( + "eventCloseChannel: the channel[{}] has been removed from the channel table before", + addrRemote); + removeItemFromTable = false; + } + + if (removeItemFromTable) { + this.channelTables.remove(addrRemote); + log.info("closeChannel: the channel[{}] was removed from channel table", addrRemote); + RemotingUtil.closeChannel(channel); + } + } + catch (Exception e) { + log.error("closeChannel: close the channel exception", e); + } + finally { + this.lockChannelTables.unlock(); + } + } + else { + log.warn("closeChannel: try to lock channel table, but timeout, {}ms", LockTimeoutMillis); + } + } + catch (InterruptedException e) { + log.error("closeChannel exception", e); + } + } + + + @Override + public void registerProcessor(int requestCode, NettyRequestProcessor processor, ExecutorService executor) { + ExecutorService executorThis = executor; + if (null == executor) { + executorThis = this.publicExecutor; + } + + Pair pair = + new Pair(processor, executorThis); + this.processorTable.put(requestCode, pair); + } + + + @Override + public RemotingCommand invokeSync(String addr, final RemotingCommand request, long timeoutMillis) + throws InterruptedException, RemotingConnectException, RemotingSendRequestException, + RemotingTimeoutException { + final Channel channel = this.getAndCreateChannel(addr); + if (channel != null && channel.isActive()) { + try { + if (this.rpcHook != null) { + this.rpcHook.doBeforeRequest(addr, request); + } + RemotingCommand response = this.invokeSyncImpl(channel, request, timeoutMillis); + if (this.rpcHook != null) { + this.rpcHook.doAfterResponse(request, response); + } + return response; + } + catch (RemotingSendRequestException e) { + log.warn("invokeSync: send request exception, so close the channel[{}]", addr); + this.closeChannel(addr, channel); + throw e; + } + catch (RemotingTimeoutException e) { + log.warn("invokeSync: wait response timeout exception, the channel[{}]", addr); + // 超时异常如果关闭连接可能会产生连锁反应 + // this.closeChannel(addr, channel); + throw e; + } + } + else { + this.closeChannel(addr, channel); + throw new RemotingConnectException(addr); + } + } + + + @Override + public void invokeAsync(String addr, RemotingCommand request, long timeoutMillis, + InvokeCallback invokeCallback) throws InterruptedException, RemotingConnectException, + RemotingTooMuchRequestException, RemotingTimeoutException, RemotingSendRequestException { + final Channel channel = this.getAndCreateChannel(addr); + if (channel != null && channel.isActive()) { + try { + if (this.rpcHook != null) { + this.rpcHook.doBeforeRequest(addr, request); + } + this.invokeAsyncImpl(channel, request, timeoutMillis, invokeCallback); + } + catch (RemotingSendRequestException e) { + log.warn("invokeAsync: send request exception, so close the channel[{}]", addr); + this.closeChannel(addr, channel); + throw e; + } + } + else { + this.closeChannel(addr, channel); + throw new RemotingConnectException(addr); + } + } + + + @Override + public void invokeOneway(String addr, RemotingCommand request, long timeoutMillis) + throws InterruptedException, RemotingConnectException, RemotingTooMuchRequestException, + RemotingTimeoutException, RemotingSendRequestException { + final Channel channel = this.getAndCreateChannel(addr); + if (channel != null && channel.isActive()) { + try { + if (this.rpcHook != null) { + this.rpcHook.doBeforeRequest(addr, request); + } + this.invokeOnewayImpl(channel, request, timeoutMillis); + } + catch (RemotingSendRequestException e) { + log.warn("invokeOneway: send request exception, so close the channel[{}]", addr); + this.closeChannel(addr, channel); + throw e; + } + } + else { + this.closeChannel(addr, channel); + throw new RemotingConnectException(addr); + } + } + + + @Override + public ExecutorService getCallbackExecutor() { + return this.publicExecutor; + } + + + @Override + public void updateNameServerAddressList(List addrs) { + List old = this.namesrvAddrList.get(); + boolean update = false; + + if (!addrs.isEmpty()) { + if (null == old) { + update = true; + } + else if (addrs.size() != old.size()) { + update = true; + } + else { + for (int i = 0; i < addrs.size() && !update; i++) { + if (!old.contains(addrs.get(i))) { + update = true; + } + } + } + + if (update) { + Collections.shuffle(addrs); + this.namesrvAddrList.set(addrs); + } + } + } + + + @Override + public ChannelEventListener getChannelEventListener() { + return channelEventListener; + } + + + public List getNamesrvAddrList() { + return namesrvAddrList.get(); + } + + + @Override + public List getNameServerAddressList() { + return this.namesrvAddrList.get(); + } + + + public RPCHook getRpcHook() { + return rpcHook; + } + + + @Override + public void registerRPCHook(RPCHook rpcHook) { + this.rpcHook = rpcHook; + } + + + @Override + public RPCHook getRPCHook() { + return this.rpcHook; + } + + + @Override + public boolean isChannelWriteable(String addr) { + ChannelWrapper cw = this.channelTables.get(addr); + if (cw != null && cw.isOK()) { + return cw.isWriteable(); + } + return true; + } +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettyRemotingServer.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettyRemotingServer.java new file mode 100644 index 000000000..9dbbdc05f --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettyRemotingServer.java @@ -0,0 +1,410 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.remoting.netty; + +import io.netty.bootstrap.ServerBootstrap; +import io.netty.buffer.PooledByteBufAllocator; +import io.netty.channel.Channel; +import io.netty.channel.ChannelDuplexHandler; +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInitializer; +import io.netty.channel.ChannelOption; +import io.netty.channel.EventLoopGroup; +import io.netty.channel.SimpleChannelInboundHandler; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.socket.SocketChannel; +import io.netty.channel.socket.nio.NioServerSocketChannel; +import io.netty.handler.timeout.IdleState; +import io.netty.handler.timeout.IdleStateEvent; +import io.netty.handler.timeout.IdleStateHandler; +import io.netty.util.concurrent.DefaultEventExecutorGroup; + +import java.net.InetSocketAddress; +import java.util.Timer; +import java.util.TimerTask; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.atomic.AtomicInteger; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.rocketmq.remoting.ChannelEventListener; +import com.alibaba.rocketmq.remoting.InvokeCallback; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.remoting.RemotingServer; +import com.alibaba.rocketmq.remoting.common.Pair; +import com.alibaba.rocketmq.remoting.common.RemotingHelper; +import com.alibaba.rocketmq.remoting.common.RemotingUtil; +import com.alibaba.rocketmq.remoting.exception.RemotingSendRequestException; +import com.alibaba.rocketmq.remoting.exception.RemotingTimeoutException; +import com.alibaba.rocketmq.remoting.exception.RemotingTooMuchRequestException; +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; + + +/** + * Remoting服务端实现 + * + * @author shijia.wxr + * @since 2013-7-13 + */ +public class NettyRemotingServer extends NettyRemotingAbstract implements RemotingServer { + private static final Logger log = LoggerFactory.getLogger(RemotingHelper.RemotingLogName); + private final ServerBootstrap serverBootstrap; + private final EventLoopGroup eventLoopGroupWorker; + private final EventLoopGroup eventLoopGroupBoss; + private final NettyServerConfig nettyServerConfig; + // 处理Callback应答器 + private final ExecutorService publicExecutor; + private final ChannelEventListener channelEventListener; + // 定时器 + private final Timer timer = new Timer("ServerHouseKeepingService", true); + private DefaultEventExecutorGroup defaultEventExecutorGroup; + + private RPCHook rpcHook; + + // 本地server绑定的端口 + private int port = 0; + + + public NettyRemotingServer(final NettyServerConfig nettyServerConfig) { + this(nettyServerConfig, null); + } + + + public NettyRemotingServer(final NettyServerConfig nettyServerConfig, + final ChannelEventListener channelEventListener) { + super(nettyServerConfig.getServerOnewaySemaphoreValue(), nettyServerConfig + .getServerAsyncSemaphoreValue()); + this.serverBootstrap = new ServerBootstrap(); + this.nettyServerConfig = nettyServerConfig; + this.channelEventListener = channelEventListener; + + int publicThreadNums = nettyServerConfig.getServerCallbackExecutorThreads(); + if (publicThreadNums <= 0) { + publicThreadNums = 4; + } + + this.publicExecutor = Executors.newFixedThreadPool(publicThreadNums, new ThreadFactory() { + private AtomicInteger threadIndex = new AtomicInteger(0); + + + @Override + public Thread newThread(Runnable r) { + return new Thread(r, "NettyServerPublicExecutor_" + this.threadIndex.incrementAndGet()); + } + }); + + this.eventLoopGroupBoss = new NioEventLoopGroup(1, new ThreadFactory() { + private AtomicInteger threadIndex = new AtomicInteger(0); + + + @Override + public Thread newThread(Runnable r) { + return new Thread(r, + String.format("NettyBossSelector_%d", this.threadIndex.incrementAndGet())); + } + }); + + this.eventLoopGroupWorker = + new NioEventLoopGroup(nettyServerConfig.getServerSelectorThreads(), new ThreadFactory() { + private AtomicInteger threadIndex = new AtomicInteger(0); + private int threadTotal = nettyServerConfig.getServerSelectorThreads(); + + + @Override + public Thread newThread(Runnable r) { + return new Thread(r, String.format("NettyServerSelector_%d_%d", threadTotal, + this.threadIndex.incrementAndGet())); + } + }); + } + + + @Override + public void start() { + this.defaultEventExecutorGroup = new DefaultEventExecutorGroup(// + nettyServerConfig.getServerWorkerThreads(), // + new ThreadFactory() { + + private AtomicInteger threadIndex = new AtomicInteger(0); + + + @Override + public Thread newThread(Runnable r) { + return new Thread(r, "NettyServerWorkerThread_" + this.threadIndex.incrementAndGet()); + } + }); + + ServerBootstrap childHandler = // + this.serverBootstrap.group(this.eventLoopGroupBoss, this.eventLoopGroupWorker) + .channel(NioServerSocketChannel.class) + // + .option(ChannelOption.SO_BACKLOG, 1024) + // + .option(ChannelOption.SO_REUSEADDR, true) + // + .option(ChannelOption.SO_KEEPALIVE, false) + // + .childOption(ChannelOption.TCP_NODELAY, true) + // + .option(ChannelOption.SO_SNDBUF, nettyServerConfig.getServerSocketSndBufSize()) + // + .option(ChannelOption.SO_RCVBUF, nettyServerConfig.getServerSocketRcvBufSize()) + // + .localAddress(new InetSocketAddress(this.nettyServerConfig.getListenPort())) + .childHandler(new ChannelInitializer() { + @Override + public void initChannel(SocketChannel ch) throws Exception { + ch.pipeline().addLast( + // + defaultEventExecutorGroup, // + new NettyEncoder(), // + new NettyDecoder(), // + new IdleStateHandler(0, 0, nettyServerConfig + .getServerChannelMaxIdleTimeSeconds()),// + new NettyConnetManageHandler(), // + new NettyServerHandler()); + } + }); + + if (nettyServerConfig.isServerPooledByteBufAllocatorEnable()) { + // 这个选项有可能会占用大量堆外内存，暂时不使用。 + childHandler.childOption(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT); + } + + try { + ChannelFuture sync = this.serverBootstrap.bind().sync(); + InetSocketAddress addr = (InetSocketAddress) sync.channel().localAddress(); + this.port = addr.getPort(); + } + catch (InterruptedException e1) { + throw new RuntimeException("this.serverBootstrap.bind().sync() InterruptedException", e1); + } + + if (this.channelEventListener != null) { + this.nettyEventExecuter.start(); + } + + // 每隔1秒扫描下异步调用超时情况 + this.timer.scheduleAtFixedRate(new TimerTask() { + + @Override + public void run() { + try { + NettyRemotingServer.this.scanResponseTable(); + } + catch (Exception e) { + log.error("scanResponseTable exception", e); + } + } + }, 1000 * 3, 1000); + } + + + @Override + public void registerProcessor(int requestCode, NettyRequestProcessor processor, ExecutorService executor) { + ExecutorService executorThis = executor; + if (null == executor) { + executorThis = this.publicExecutor; + } + + Pair pair = + new Pair(processor, executorThis); + this.processorTable.put(requestCode, pair); + } + + + @Override + public void registerDefaultProcessor(NettyRequestProcessor processor, ExecutorService executor) { + this.defaultRequestProcessor = new Pair(processor, executor); + } + + + @Override + public RemotingCommand invokeSync(final Channel channel, final RemotingCommand request, + final long timeoutMillis) throws InterruptedException, RemotingSendRequestException, + RemotingTimeoutException { + return this.invokeSyncImpl(channel, request, timeoutMillis); + } + + + @Override + public void invokeAsync(Channel channel, RemotingCommand request, long timeoutMillis, + InvokeCallback invokeCallback) throws InterruptedException, RemotingTooMuchRequestException, + RemotingTimeoutException, RemotingSendRequestException { + this.invokeAsyncImpl(channel, request, timeoutMillis, invokeCallback); + } + + + @Override + public void invokeOneway(Channel channel, RemotingCommand request, long timeoutMillis) + throws InterruptedException, RemotingTooMuchRequestException, RemotingTimeoutException, + RemotingSendRequestException { + this.invokeOnewayImpl(channel, request, timeoutMillis); + } + + + @Override + public void shutdown() { + try { + if (this.timer != null) { + this.timer.cancel(); + } + + this.eventLoopGroupBoss.shutdownGracefully(); + + this.eventLoopGroupWorker.shutdownGracefully(); + + if (this.nettyEventExecuter != null) { + this.nettyEventExecuter.shutdown(); + } + + if (this.defaultEventExecutorGroup != null) { + this.defaultEventExecutorGroup.shutdownGracefully(); + } + } + catch (Exception e) { + log.error("NettyRemotingServer shutdown exception, ", e); + } + + if (this.publicExecutor != null) { + try { + this.publicExecutor.shutdown(); + } + catch (Exception e) { + log.error("NettyRemotingServer shutdown exception, ", e); + } + } + } + + + @Override + public ChannelEventListener getChannelEventListener() { + return channelEventListener; + } + + + @Override + public ExecutorService getCallbackExecutor() { + return this.publicExecutor; + } + + class NettyServerHandler extends SimpleChannelInboundHandler { + + @Override + protected void channelRead0(ChannelHandlerContext ctx, RemotingCommand msg) throws Exception { + processMessageReceived(ctx, msg); + } + } + + class NettyConnetManageHandler extends ChannelDuplexHandler { + @Override + public void channelRegistered(ChannelHandlerContext ctx) throws Exception { + final String remoteAddress = RemotingHelper.parseChannelRemoteAddr(ctx.channel()); + log.info("NETTY SERVER PIPELINE: channelRegistered {}", remoteAddress); + super.channelRegistered(ctx); + } + + + @Override + public void channelUnregistered(ChannelHandlerContext ctx) throws Exception { + final String remoteAddress = RemotingHelper.parseChannelRemoteAddr(ctx.channel()); + log.info("NETTY SERVER PIPELINE: channelUnregistered, the channel[{}]", remoteAddress); + super.channelUnregistered(ctx); + } + + + @Override + public void channelActive(ChannelHandlerContext ctx) throws Exception { + final String remoteAddress = RemotingHelper.parseChannelRemoteAddr(ctx.channel()); + log.info("NETTY SERVER PIPELINE: channelActive, the channel[{}]", remoteAddress); + super.channelActive(ctx); + + if (NettyRemotingServer.this.channelEventListener != null) { + NettyRemotingServer.this.putNettyEvent(new NettyEvent(NettyEventType.CONNECT, remoteAddress + .toString(), ctx.channel())); + } + } + + + @Override + public void channelInactive(ChannelHandlerContext ctx) throws Exception { + final String remoteAddress = RemotingHelper.parseChannelRemoteAddr(ctx.channel()); + log.info("NETTY SERVER PIPELINE: channelInactive, the channel[{}]", remoteAddress); + super.channelInactive(ctx); + + if (NettyRemotingServer.this.channelEventListener != null) { + NettyRemotingServer.this.putNettyEvent(new NettyEvent(NettyEventType.CLOSE, remoteAddress + .toString(), ctx.channel())); + } + } + + + @Override + public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception { + if (evt instanceof IdleStateEvent) { + IdleStateEvent evnet = (IdleStateEvent) evt; + if (evnet.state().equals(IdleState.ALL_IDLE)) { + final String remoteAddress = RemotingHelper.parseChannelRemoteAddr(ctx.channel()); + log.warn("NETTY SERVER PIPELINE: IDLE exception [{}]", remoteAddress); + RemotingUtil.closeChannel(ctx.channel()); + if (NettyRemotingServer.this.channelEventListener != null) { + NettyRemotingServer.this.putNettyEvent(new NettyEvent(NettyEventType.IDLE, + remoteAddress.toString(), ctx.channel())); + } + } + } + + ctx.fireUserEventTriggered(evt); + } + + + @Override + public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { + final String remoteAddress = RemotingHelper.parseChannelRemoteAddr(ctx.channel()); + log.warn("NETTY SERVER PIPELINE: exceptionCaught {}", remoteAddress); + log.warn("NETTY SERVER PIPELINE: exceptionCaught exception.", cause); + + if (NettyRemotingServer.this.channelEventListener != null) { + NettyRemotingServer.this.putNettyEvent(new NettyEvent(NettyEventType.EXCEPTION, remoteAddress + .toString(), ctx.channel())); + } + + RemotingUtil.closeChannel(ctx.channel()); + } + } + + + @Override + public void registerRPCHook(RPCHook rpcHook) { + this.rpcHook = rpcHook; + } + + + @Override + public RPCHook getRPCHook() { + return this.rpcHook; + } + + + @Override + public int localListenPort() { + return this.port; + } +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettyRequestProcessor.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettyRequestProcessor.java new file mode 100644 index 000000000..f817beb04 --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettyRequestProcessor.java @@ -0,0 +1,32 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.remoting.netty; + +import io.netty.channel.ChannelHandlerContext; + +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; + + +/** + * 接收请求处理器，服务器与客户端通用 + * + * @author shijia.wxr + * @since 2013-7-13 + */ +public interface NettyRequestProcessor { + public RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand request) + throws Exception; +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettyServerConfig.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettyServerConfig.java new file mode 100644 index 000000000..6b8067d4c --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettyServerConfig.java @@ -0,0 +1,136 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.remoting.netty; + +/** + * Netty服务端配置 + * + * @author shijia.wxr + * @since 2013-7-13 + */ +public class NettyServerConfig { + private int listenPort = 8888; + private int serverWorkerThreads = 8; + private int serverCallbackExecutorThreads = 0; + private int serverSelectorThreads = 3; + private int serverOnewaySemaphoreValue = 256; + private int serverAsyncSemaphoreValue = 64; + private int serverChannelMaxIdleTimeSeconds = 120; + + private int serverSocketSndBufSize = NettySystemConfig.SocketSndbufSize; + private int serverSocketRcvBufSize = NettySystemConfig.SocketRcvbufSize; + private boolean serverPooledByteBufAllocatorEnable = false; + + + public int getListenPort() { + return listenPort; + } + + + public void setListenPort(int listenPort) { + this.listenPort = listenPort; + } + + + public int getServerWorkerThreads() { + return serverWorkerThreads; + } + + + public void setServerWorkerThreads(int serverWorkerThreads) { + this.serverWorkerThreads = serverWorkerThreads; + } + + + public int getServerSelectorThreads() { + return serverSelectorThreads; + } + + + public void setServerSelectorThreads(int serverSelectorThreads) { + this.serverSelectorThreads = serverSelectorThreads; + } + + + public int getServerOnewaySemaphoreValue() { + return serverOnewaySemaphoreValue; + } + + + public void setServerOnewaySemaphoreValue(int serverOnewaySemaphoreValue) { + this.serverOnewaySemaphoreValue = serverOnewaySemaphoreValue; + } + + + public int getServerCallbackExecutorThreads() { + return serverCallbackExecutorThreads; + } + + + public void setServerCallbackExecutorThreads(int serverCallbackExecutorThreads) { + this.serverCallbackExecutorThreads = serverCallbackExecutorThreads; + } + + + public int getServerAsyncSemaphoreValue() { + return serverAsyncSemaphoreValue; + } + + + public void setServerAsyncSemaphoreValue(int serverAsyncSemaphoreValue) { + this.serverAsyncSemaphoreValue = serverAsyncSemaphoreValue; + } + + + public int getServerChannelMaxIdleTimeSeconds() { + return serverChannelMaxIdleTimeSeconds; + } + + + public void setServerChannelMaxIdleTimeSeconds(int serverChannelMaxIdleTimeSeconds) { + this.serverChannelMaxIdleTimeSeconds = serverChannelMaxIdleTimeSeconds; + } + + + public int getServerSocketSndBufSize() { + return serverSocketSndBufSize; + } + + + public void setServerSocketSndBufSize(int serverSocketSndBufSize) { + this.serverSocketSndBufSize = serverSocketSndBufSize; + } + + + public int getServerSocketRcvBufSize() { + return serverSocketRcvBufSize; + } + + + public void setServerSocketRcvBufSize(int serverSocketRcvBufSize) { + this.serverSocketRcvBufSize = serverSocketRcvBufSize; + } + + + public boolean isServerPooledByteBufAllocatorEnable() { + return serverPooledByteBufAllocatorEnable; + } + + + public void setServerPooledByteBufAllocatorEnable(boolean serverPooledByteBufAllocatorEnable) { + this.serverPooledByteBufAllocatorEnable = serverPooledByteBufAllocatorEnable; + } +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettySystemConfig.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettySystemConfig.java new file mode 100644 index 000000000..ad994ef33 --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/NettySystemConfig.java @@ -0,0 +1,29 @@ +package com.alibaba.rocketmq.remoting.netty; + +public class NettySystemConfig { + public static final String SystemPropertyNettyPooledByteBufAllocatorEnable = + "com.rocketmq.remoting.nettyPooledByteBufAllocatorEnable"; + public static boolean NettyPooledByteBufAllocatorEnable = // + Boolean + .parseBoolean(System.getProperty(SystemPropertyNettyPooledByteBufAllocatorEnable, "false")); + + public static final String SystemPropertySocketSndbufSize = // + "com.rocketmq.remoting.socket.sndbuf.size"; + public static int SocketSndbufSize = // + Integer.parseInt(System.getProperty(SystemPropertySocketSndbufSize, "65535")); + + public static final String SystemPropertySocketRcvbufSize = // + "com.rocketmq.remoting.socket.rcvbuf.size"; + public static int SocketRcvbufSize = // + Integer.parseInt(System.getProperty(SystemPropertySocketRcvbufSize, "65535")); + + public static final String SystemPropertyClientAsyncSemaphoreValue = // + "com.rocketmq.remoting.clientAsyncSemaphoreValue"; + public static int ClientAsyncSemaphoreValue = // + Integer.parseInt(System.getProperty(SystemPropertyClientAsyncSemaphoreValue, "2048")); + + public static final String SystemPropertyClientOnewaySemaphoreValue = // + "com.rocketmq.remoting.clientOnewaySemaphoreValue"; + public static int ClientOnewaySemaphoreValue = // + Integer.parseInt(System.getProperty(SystemPropertyClientOnewaySemaphoreValue, "2048")); +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/ResponseFuture.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/ResponseFuture.java new file mode 100644 index 000000000..e9c896a9d --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/netty/ResponseFuture.java @@ -0,0 +1,150 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.remoting.netty; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import com.alibaba.rocketmq.remoting.InvokeCallback; +import com.alibaba.rocketmq.remoting.common.SemaphoreReleaseOnlyOnce; +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; + + +/** + * 异步请求应答封装 + * + * @author shijia.wxr + * @since 2013-7-13 + */ +public class ResponseFuture { + private volatile RemotingCommand responseCommand; + private volatile boolean sendRequestOK = true; + private volatile Throwable cause; + private final int opaque; + private final long timeoutMillis; + private final InvokeCallback invokeCallback; + private final long beginTimestamp = System.currentTimeMillis(); + private final CountDownLatch countDownLatch = new CountDownLatch(1); + + // 保证信号量至多至少只被释放一次 + private final SemaphoreReleaseOnlyOnce once; + + // 保证回调的callback方法至多至少只被执行一次 + private final AtomicBoolean executeCallbackOnlyOnce = new AtomicBoolean(false); + + + public ResponseFuture(int opaque, long timeoutMillis, InvokeCallback invokeCallback, + SemaphoreReleaseOnlyOnce once) { + this.opaque = opaque; + this.timeoutMillis = timeoutMillis; + this.invokeCallback = invokeCallback; + this.once = once; + } + + + public void executeInvokeCallback() { + if (invokeCallback != null) { + if (this.executeCallbackOnlyOnce.compareAndSet(false, true)) { + invokeCallback.operationComplete(this); + } + } + } + + + public void release() { + if (this.once != null) { + this.once.release(); + } + } + + + public boolean isTimeout() { + long diff = System.currentTimeMillis() - this.beginTimestamp; + return diff > this.timeoutMillis; + } + + + public RemotingCommand waitResponse(final long timeoutMillis) throws InterruptedException { + this.countDownLatch.await(timeoutMillis, TimeUnit.MILLISECONDS); + return this.responseCommand; + } + + + public void putResponse(final RemotingCommand responseCommand) { + this.responseCommand = responseCommand; + this.countDownLatch.countDown(); + } + + + public long getBeginTimestamp() { + return beginTimestamp; + } + + + public boolean isSendRequestOK() { + return sendRequestOK; + } + + + public void setSendRequestOK(boolean sendRequestOK) { + this.sendRequestOK = sendRequestOK; + } + + + public long getTimeoutMillis() { + return timeoutMillis; + } + + + public InvokeCallback getInvokeCallback() { + return invokeCallback; + } + + + public Throwable getCause() { + return cause; + } + + + public void setCause(Throwable cause) { + this.cause = cause; + } + + + public RemotingCommand getResponseCommand() { + return responseCommand; + } + + + public void setResponseCommand(RemotingCommand responseCommand) { + this.responseCommand = responseCommand; + } + + + public int getOpaque() { + return opaque; + } + + + @Override + public String toString() { + return "ResponseFuture [responseCommand=" + responseCommand + ", sendRequestOK=" + sendRequestOK + + ", cause=" + cause + ", opaque=" + opaque + ", timeoutMillis=" + timeoutMillis + + ", invokeCallback=" + invokeCallback + ", beginTimestamp=" + beginTimestamp + + ", countDownLatch=" + countDownLatch + "]"; + } +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/protocol/LanguageCode.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/protocol/LanguageCode.java new file mode 100644 index 000000000..1ec089841 --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/protocol/LanguageCode.java @@ -0,0 +1,12 @@ +package com.alibaba.rocketmq.remoting.protocol; + +public enum LanguageCode { + JAVA, + CPP, + DOTNET, + PYTHON, + DELPHI, + ERLANG, + RUBY, + OTHER, +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/protocol/RemotingCommand.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/protocol/RemotingCommand.java new file mode 100644 index 000000000..486cd3201 --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/protocol/RemotingCommand.java @@ -0,0 +1,506 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.remoting.protocol; + +import java.lang.annotation.Annotation; +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.concurrent.atomic.AtomicInteger; + +import com.alibaba.fastjson.annotation.JSONField; +import com.alibaba.rocketmq.remoting.CommandCustomHeader; +import com.alibaba.rocketmq.remoting.annotation.CFNotNull; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; + + +/** + * Remoting模块中，服务器与客户端通过传递RemotingCommand来交互 + * + * @author shijia.wxr + * @since 2013-7-13 + */ +public class RemotingCommand { + public static String RemotingVersionKey = "rocketmq.remoting.version"; + private static volatile int ConfigVersion = -1; + private static AtomicInteger RequestId = new AtomicInteger(0); + + private static final int RPC_TYPE = 0; // 0, REQUEST_COMMAND + // 1, RESPONSE_COMMAND + + private static final int RPC_ONEWAY = 1; // 0, RPC + // 1, Oneway + + /** + * Header 部分 + */ + private int code; + private LanguageCode language = LanguageCode.JAVA; + private int version = 0; + private int opaque = RequestId.getAndIncrement(); + private int flag = 0; + private String remark; + private HashMap extFields; + + private transient CommandCustomHeader customHeader; + + /** + * Body 部分 + */ + private transient byte[] body; + + + protected RemotingCommand() { + } + + + public static RemotingCommand createRequestCommand(int code, CommandCustomHeader customHeader) { + RemotingCommand cmd = new RemotingCommand(); + cmd.setCode(code); + cmd.customHeader = customHeader; + setCmdVersion(cmd); + return cmd; + } + + + public static RemotingCommand createResponseCommand(Class classHeader) { + RemotingCommand cmd = + createResponseCommand(RemotingSysResponseCode.SYSTEM_ERROR, "not set any response code", + classHeader); + + return cmd; + } + + + public static RemotingCommand createResponseCommand(int code, String remark) { + return createResponseCommand(code, remark, null); + } + + + /** + * 只有通信层内部会调用，业务不会调用 + */ + public static RemotingCommand createResponseCommand(int code, String remark, + Class classHeader) { + RemotingCommand cmd = new RemotingCommand(); + cmd.markResponseType(); + cmd.setCode(code); + cmd.setRemark(remark); + setCmdVersion(cmd); + + if (classHeader != null) { + try { + CommandCustomHeader objectHeader = classHeader.newInstance(); + cmd.customHeader = objectHeader; + } + catch (InstantiationException e) { + return null; + } + catch (IllegalAccessException e) { + return null; + } + } + + return cmd; + } + + + private static void setCmdVersion(RemotingCommand cmd) { + if (ConfigVersion >= 0) { + cmd.setVersion(ConfigVersion); + } + else { + String v = System.getProperty(RemotingVersionKey); + if (v != null) { + int value = Integer.parseInt(v); + cmd.setVersion(value); + ConfigVersion = value; + } + } + } + + + public void makeCustomHeaderToNet() { + if (this.customHeader != null) { + Field[] fields = this.customHeader.getClass().getDeclaredFields(); + if (null == this.extFields) { + this.extFields = new HashMap(); + } + + for (Field field : fields) { + if (!Modifier.isStatic(field.getModifiers())) { + String name = field.getName(); + if (!name.startsWith("this")) { + Object value = null; + try { + field.setAccessible(true); + value = field.get(this.customHeader); + } + catch (IllegalArgumentException e) { + } + catch (IllegalAccessException e) { + } + + if (value != null) { + this.extFields.put(name, value.toString()); + } + } + } + } + } + } + + + public CommandCustomHeader readCustomHeader() { + return customHeader; + } + + + public void writeCustomHeader(CommandCustomHeader customHeader) { + this.customHeader = customHeader; + } + + private static final String StringName = String.class.getCanonicalName();// + + private static final String IntegerName1 = Integer.class.getCanonicalName();// + private static final String IntegerName2 = int.class.getCanonicalName();// + + private static final String LongName1 = Long.class.getCanonicalName();// + private static final String LongName2 = long.class.getCanonicalName();// + + private static final String BooleanName1 = Boolean.class.getCanonicalName();// + private static final String BooleanName2 = boolean.class.getCanonicalName();// + + private static final String DoubleName1 = Double.class.getCanonicalName();// + private static final String DoubleName2 = double.class.getCanonicalName();// + + + public CommandCustomHeader decodeCommandCustomHeader(Class classHeader) + throws RemotingCommandException { + if (this.extFields != null) { + CommandCustomHeader objectHeader; + try { + objectHeader = classHeader.newInstance(); + } + catch (InstantiationException e) { + return null; + } + catch (IllegalAccessException e) { + return null; + } + + // 检查返回对象是否有效 + Field[] fields = objectHeader.getClass().getDeclaredFields(); + for (Field field : fields) { + if (!Modifier.isStatic(field.getModifiers())) { + String fieldName = field.getName(); + if (!fieldName.startsWith("this")) { + try { + String value = this.extFields.get(fieldName); + if (null == value) { + Annotation annotation = field.getAnnotation(CFNotNull.class); + if (annotation != null) { + throw new RemotingCommandException("the custom field <" + fieldName + + "> is null"); + } + + continue; + } + + field.setAccessible(true); + String type = field.getType().getCanonicalName(); + Object valueParsed = null; + + if (type.equals(StringName)) { + valueParsed = value; + } + else if (type.equals(IntegerName1) || type.equals(IntegerName2)) { + valueParsed = Integer.parseInt(value); + } + else if (type.equals(LongName1) || type.equals(LongName2)) { + valueParsed = Long.parseLong(value); + } + else if (type.equals(BooleanName1) || type.equals(BooleanName2)) { + valueParsed = Boolean.parseBoolean(value); + } + else if (type.equals(DoubleName1) || type.equals(DoubleName2)) { + valueParsed = Double.parseDouble(value); + } + else { + throw new RemotingCommandException("the custom field <" + fieldName + + "> type is not supported"); + } + + field.set(objectHeader, valueParsed); + + } + catch (Throwable e) { + } + } + } + } + + objectHeader.checkFields(); + + return objectHeader; + } + + return null; + } + + + private byte[] buildHeader() { + this.makeCustomHeaderToNet(); + return RemotingSerializable.encode(this); + } + + + public ByteBuffer encode() { + // 1> header length size + int length = 4; + + // 2> header data length + byte[] headerData = this.buildHeader(); + length += headerData.length; + + // 3> body data length + if (this.body != null) { + length += body.length; + } + + ByteBuffer result = ByteBuffer.allocate(4 + length); + + // length + result.putInt(length); + + // header length + result.putInt(headerData.length); + + // header data + result.put(headerData); + + // body data; + if (this.body != null) { + result.put(this.body); + } + + result.flip(); + + return result; + } + + + public ByteBuffer encodeHeader() { + return encodeHeader(this.body != null ? this.body.length : 0); + } + + + /** + * 只打包Header，body部分独立传输 + */ + public ByteBuffer encodeHeader(final int bodyLength) { + // 1> header length size + int length = 4; + + // 2> header data length + byte[] headerData = this.buildHeader(); + length += headerData.length; + + // 3> body data length + length += bodyLength; + + ByteBuffer result = ByteBuffer.allocate(4 + length - bodyLength); + + // length + result.putInt(length); + + // header length + result.putInt(headerData.length); + + // header data + result.put(headerData); + + result.flip(); + + return result; + } + + + public static RemotingCommand decode(final byte[] array) { + ByteBuffer byteBuffer = ByteBuffer.wrap(array); + return decode(byteBuffer); + } + + + public static RemotingCommand decode(final ByteBuffer byteBuffer) { + int length = byteBuffer.limit(); + int headerLength = byteBuffer.getInt(); + + byte[] headerData = new byte[headerLength]; + byteBuffer.get(headerData); + + int bodyLength = length - 4 - headerLength; + byte[] bodyData = null; + if (bodyLength > 0) { + bodyData = new byte[bodyLength]; + byteBuffer.get(bodyData); + } + + RemotingCommand cmd = RemotingSerializable.decode(headerData, RemotingCommand.class); + cmd.body = bodyData; + + return cmd; + } + + + public void markResponseType() { + int bits = 1 << RPC_TYPE; + this.flag |= bits; + } + + + @JSONField(serialize = false) + public boolean isResponseType() { + int bits = 1 << RPC_TYPE; + return (this.flag & bits) == bits; + } + + + public void markOnewayRPC() { + int bits = 1 << RPC_ONEWAY; + this.flag |= bits; + } + + + @JSONField(serialize = false) + public boolean isOnewayRPC() { + int bits = 1 << RPC_ONEWAY; + return (this.flag & bits) == bits; + } + + + public int getCode() { + return code; + } + + + public void setCode(int code) { + this.code = code; + } + + + @JSONField(serialize = false) + public RemotingCommandType getType() { + if (this.isResponseType()) { + return RemotingCommandType.RESPONSE_COMMAND; + } + + return RemotingCommandType.REQUEST_COMMAND; + } + + + public LanguageCode getLanguage() { + return language; + } + + + public void setLanguage(LanguageCode language) { + this.language = language; + } + + + public int getVersion() { + return version; + } + + + public void setVersion(int version) { + this.version = version; + } + + + public int getOpaque() { + return opaque; + } + + + public void setOpaque(int opaque) { + this.opaque = opaque; + } + + + public int getFlag() { + return flag; + } + + + public void setFlag(int flag) { + this.flag = flag; + } + + + public String getRemark() { + return remark; + } + + + public void setRemark(String remark) { + this.remark = remark; + } + + + public byte[] getBody() { + return body; + } + + + public void setBody(byte[] body) { + this.body = body; + } + + + public HashMap getExtFields() { + return extFields; + } + + + public void setExtFields(HashMap extFields) { + this.extFields = extFields; + } + + + public static int createNewRequestId() { + return RequestId.incrementAndGet(); + } + + + public void addExtField(String key, String value) { + if (null == extFields) { + extFields = new HashMap(); + } + extFields.put(key, value); + } + + + @Override + public String toString() { + return "RemotingCommand [code=" + code + ", language=" + language + ", version=" + version + + ", opaque=" + opaque + ", flag(B)=" + Integer.toBinaryString(flag) + ", remark=" + remark + + ", extFields=" + extFields + "]"; + } + +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/protocol/RemotingCommandType.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/protocol/RemotingCommandType.java new file mode 100644 index 000000000..d287d1453 --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/protocol/RemotingCommandType.java @@ -0,0 +1,27 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.remoting.protocol; + +/** + * 标识RemotingCommand是请求还是应答类型 + * + * @author shijia.wxr + * @since 2013-7-13 + */ +public enum RemotingCommandType { + REQUEST_COMMAND, + RESPONSE_COMMAND; +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/protocol/RemotingSerializable.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/protocol/RemotingSerializable.java new file mode 100644 index 000000000..037e34b2a --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/protocol/RemotingSerializable.java @@ -0,0 +1,72 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.remoting.protocol; + +import java.nio.charset.Charset; + +import com.alibaba.fastjson.JSON; + + +/** + * 复杂对象的序列化，利用json来实现 + * + * @author shijia.wxr + * @since 2013-7-13 + */ +public abstract class RemotingSerializable { + public String toJson() { + return toJson(false); + } + + + public String toJson(final boolean prettyFormat) { + return toJson(this, prettyFormat); + } + + + public static String toJson(final Object obj, boolean prettyFormat) { + return JSON.toJSONString(obj, prettyFormat); + } + + + public static T fromJson(String json, Class classOfT) { + return JSON.parseObject(json, classOfT); + } + + + public byte[] encode() { + final String json = this.toJson(); + if (json != null) { + return json.getBytes(); + } + return null; + } + + + public static byte[] encode(final Object obj) { + final String json = toJson(obj, false); + if (json != null) { + return json.getBytes(Charset.forName("UTF-8")); + } + return null; + } + + + public static T decode(final byte[] data, Class classOfT) { + final String json = new String(data, Charset.forName("UTF-8")); + return fromJson(json, classOfT); + } +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/protocol/RemotingSysResponseCode.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/protocol/RemotingSysResponseCode.java new file mode 100644 index 000000000..2c0aa67e6 --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/protocol/RemotingSysResponseCode.java @@ -0,0 +1,12 @@ +package com.alibaba.rocketmq.remoting.protocol; + +public class RemotingSysResponseCode { + // 成功 + public static final int SUCCESS = 0; + // 发生了未捕获异常 + public static final int SYSTEM_ERROR = 1; + // 由于线程池拥堵，系统繁忙 + public static final int SYSTEM_BUSY = 2; + // 请求代码不支持 + public static final int REQUEST_CODE_NOT_SUPPORTED = 3; +} diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/protocol/protocol.txt b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/protocol/protocol.txt new file mode 100644 index 000000000..a660b16f8 --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/protocol/protocol.txt @@ -0,0 +1,12 @@ +// +// Remoting通信协议 +// 2013-01-21 19:11:14 誓嘉草拟 V0.1 +// +// 协议格式

+// 1 2 3 4 +// 协议分4部分，含义分别如下 +// 1、大端4个字节整数，等于2、3、4长度总和 +// 2、大端4个字节整数，等于3的长度 +// 3、使用json序列化数据 +// 4、应用自定义二进制序列化数据 +// diff --git a/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/version/V3_1_9.java b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/version/V3_1_9.java new file mode 100644 index 000000000..4c2ff8ded --- /dev/null +++ b/rocketmq-remoting/src/main/java/com/alibaba/rocketmq/remoting/version/V3_1_9.java @@ -0,0 +1,5 @@ +package com.alibaba.rocketmq.remoting.version; + +public class V3_1_9 { + +} diff --git a/rocketmq-remoting/src/test/java/com/alibaba/rocketmq/remoting/ExceptionTest.java b/rocketmq-remoting/src/test/java/com/alibaba/rocketmq/remoting/ExceptionTest.java new file mode 100644 index 000000000..c6239c8b4 --- /dev/null +++ b/rocketmq-remoting/src/test/java/com/alibaba/rocketmq/remoting/ExceptionTest.java @@ -0,0 +1,83 @@ +/** + * $Id: ExceptionTest.java 1831 2013-05-16 01:39:51Z shijia.wxr $ + */ +package com.alibaba.rocketmq.remoting; + +import static org.junit.Assert.assertTrue; +import io.netty.channel.ChannelHandlerContext; + +import java.util.concurrent.Executors; + +import org.junit.Test; + +import com.alibaba.rocketmq.remoting.exception.RemotingConnectException; +import com.alibaba.rocketmq.remoting.exception.RemotingSendRequestException; +import com.alibaba.rocketmq.remoting.exception.RemotingTimeoutException; +import com.alibaba.rocketmq.remoting.netty.NettyClientConfig; +import com.alibaba.rocketmq.remoting.netty.NettyRemotingClient; +import com.alibaba.rocketmq.remoting.netty.NettyRemotingServer; +import com.alibaba.rocketmq.remoting.netty.NettyRequestProcessor; +import com.alibaba.rocketmq.remoting.netty.NettyServerConfig; +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; + + +/** + * @author shijia.wxr + */ +public class ExceptionTest { + private static RemotingClient createRemotingClient() { + NettyClientConfig config = new NettyClientConfig(); + RemotingClient client = new NettyRemotingClient(config); + client.start(); + return client; + } + + + private static RemotingServer createRemotingServer() throws InterruptedException { + NettyServerConfig config = new NettyServerConfig(); + RemotingServer client = new NettyRemotingServer(config); + client.registerProcessor(0, new NettyRequestProcessor() { + private int i = 0; + + + @Override + public RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand request) { + System.out.println("processRequest=" + request + " " + (i++)); + request.setRemark("hello, I am respponse " + ctx.channel().remoteAddress()); + return request; + } + }, Executors.newCachedThreadPool()); + client.start(); + return client; + } + + + @Test + public void test_CONNECT_EXCEPTION() { + RemotingClient client = createRemotingClient(); + + RemotingCommand request = RemotingCommand.createRequestCommand(0, null); + RemotingCommand response = null; + try { + response = client.invokeSync("localhost:8888", request, 1000 * 3); + } + catch (RemotingConnectException e) { + e.printStackTrace(); + } + catch (RemotingSendRequestException e) { + e.printStackTrace(); + } + catch (RemotingTimeoutException e) { + e.printStackTrace(); + } + catch (InterruptedException e) { + e.printStackTrace(); + } + System.out.println("invoke result = " + response); + assertTrue(null == response); + + client.shutdown(); + System.out.println("-----------------------------------------------------------------"); + } + +} diff --git a/rocketmq-remoting/src/test/java/com/alibaba/rocketmq/remoting/MixTest.java b/rocketmq-remoting/src/test/java/com/alibaba/rocketmq/remoting/MixTest.java new file mode 100644 index 000000000..57d318db0 --- /dev/null +++ b/rocketmq-remoting/src/test/java/com/alibaba/rocketmq/remoting/MixTest.java @@ -0,0 +1,17 @@ +/** + * $Id: MixTest.java 1831 2013-05-16 01:39:51Z shijia.wxr $ + */ +package com.alibaba.rocketmq.remoting; + +import org.junit.Test; + + +/** + * @author shijia.wxr + */ +public class MixTest { + @Test + public void test_extFieldsValue() { + + } +} diff --git a/rocketmq-remoting/src/test/java/com/alibaba/rocketmq/remoting/NettyConnectionTest.java b/rocketmq-remoting/src/test/java/com/alibaba/rocketmq/remoting/NettyConnectionTest.java new file mode 100644 index 000000000..f7072b158 --- /dev/null +++ b/rocketmq-remoting/src/test/java/com/alibaba/rocketmq/remoting/NettyConnectionTest.java @@ -0,0 +1,47 @@ +package com.alibaba.rocketmq.remoting; + +import org.junit.Test; + +import com.alibaba.rocketmq.remoting.exception.RemotingConnectException; +import com.alibaba.rocketmq.remoting.exception.RemotingSendRequestException; +import com.alibaba.rocketmq.remoting.exception.RemotingTimeoutException; +import com.alibaba.rocketmq.remoting.netty.NettyClientConfig; +import com.alibaba.rocketmq.remoting.netty.NettyRemotingClient; +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; + + +/** + * 连接超时测试 + * + * @author shijia.wxr + * @since 2013-7-6 + */ +public class NettyConnectionTest { + public static RemotingClient createRemotingClient() { + NettyClientConfig config = new NettyClientConfig(); + config.setClientChannelMaxIdleTimeSeconds(15); + RemotingClient client = new NettyRemotingClient(config); + client.start(); + return client; + } + + + @Test + public void test_connect_timeout() throws InterruptedException, RemotingConnectException, + RemotingSendRequestException, RemotingTimeoutException { + RemotingClient client = createRemotingClient(); + + for (int i = 0; i < 100; i++) { + try { + RemotingCommand request = RemotingCommand.createRequestCommand(0, null); + RemotingCommand response = client.invokeSync("localhost:8888", request, 1000 * 3); + } + catch (Exception e) { + e.printStackTrace(); + } + } + + client.shutdown(); + System.out.println("-----------------------------------------------------------------"); + } +} diff --git a/rocketmq-remoting/src/test/java/com/alibaba/rocketmq/remoting/NettyIdleTest.java b/rocketmq-remoting/src/test/java/com/alibaba/rocketmq/remoting/NettyIdleTest.java new file mode 100644 index 000000000..24bcadd88 --- /dev/null +++ b/rocketmq-remoting/src/test/java/com/alibaba/rocketmq/remoting/NettyIdleTest.java @@ -0,0 +1,75 @@ +package com.alibaba.rocketmq.remoting; + +import static org.junit.Assert.assertTrue; +import io.netty.channel.ChannelHandlerContext; + +import java.util.concurrent.Executors; + +import com.alibaba.rocketmq.remoting.exception.RemotingConnectException; +import com.alibaba.rocketmq.remoting.exception.RemotingSendRequestException; +import com.alibaba.rocketmq.remoting.exception.RemotingTimeoutException; +import com.alibaba.rocketmq.remoting.netty.NettyClientConfig; +import com.alibaba.rocketmq.remoting.netty.NettyRemotingClient; +import com.alibaba.rocketmq.remoting.netty.NettyRemotingServer; +import com.alibaba.rocketmq.remoting.netty.NettyRequestProcessor; +import com.alibaba.rocketmq.remoting.netty.NettyServerConfig; +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; + + +/** + * @author shijia.wxr + * @since 2013-7-6 + */ +public class NettyIdleTest { + public static RemotingClient createRemotingClient() { + NettyClientConfig config = new NettyClientConfig(); + config.setClientChannelMaxIdleTimeSeconds(15); + RemotingClient client = new NettyRemotingClient(config); + client.start(); + return client; + } + + + public static RemotingServer createRemotingServer() throws InterruptedException { + NettyServerConfig config = new NettyServerConfig(); + config.setServerChannelMaxIdleTimeSeconds(30); + RemotingServer remotingServer = new NettyRemotingServer(config); + remotingServer.registerProcessor(0, new NettyRequestProcessor() { + private int i = 0; + + + @Override + public RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand request) { + System.out.println("processRequest=" + request + " " + (i++)); + request.setRemark("hello, I am respponse " + ctx.channel().remoteAddress()); + return request; + } + }, Executors.newCachedThreadPool()); + remotingServer.start(); + return remotingServer; + } + + + // @Test + public void test_idle_event() throws InterruptedException, RemotingConnectException, + RemotingSendRequestException, RemotingTimeoutException { + RemotingServer server = createRemotingServer(); + RemotingClient client = createRemotingClient(); + + for (int i = 0; i < 10; i++) { + RemotingCommand request = RemotingCommand.createRequestCommand(0, null); + RemotingCommand response = client.invokeSync("localhost:8888", request, 1000 * 3); + System.out.println(i + " invoke result = " + response); + assertTrue(response != null); + + Thread.sleep(1000 * 10); + } + + Thread.sleep(1000 * 60); + + client.shutdown(); + server.shutdown(); + System.out.println("-----------------------------------------------------------------"); + } + +} diff --git a/rocketmq-remoting/src/test/java/com/alibaba/rocketmq/remoting/NettyRPCTest.java b/rocketmq-remoting/src/test/java/com/alibaba/rocketmq/remoting/NettyRPCTest.java new file mode 100644 index 000000000..5110513e1 --- /dev/null +++ b/rocketmq-remoting/src/test/java/com/alibaba/rocketmq/remoting/NettyRPCTest.java @@ -0,0 +1,240 @@ +/** + * $Id: NettyRPCTest.java 1831 2013-05-16 01:39:51Z shijia.wxr $ + */ +package com.alibaba.rocketmq.remoting; + +import static org.junit.Assert.assertTrue; +import io.netty.channel.ChannelHandlerContext; + +import java.util.concurrent.Executors; + +import org.junit.Test; + +import com.alibaba.rocketmq.remoting.annotation.CFNullable; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; +import com.alibaba.rocketmq.remoting.exception.RemotingConnectException; +import com.alibaba.rocketmq.remoting.exception.RemotingSendRequestException; +import com.alibaba.rocketmq.remoting.exception.RemotingTimeoutException; +import com.alibaba.rocketmq.remoting.exception.RemotingTooMuchRequestException; +import com.alibaba.rocketmq.remoting.netty.NettyClientConfig; +import com.alibaba.rocketmq.remoting.netty.NettyRemotingClient; +import com.alibaba.rocketmq.remoting.netty.NettyRemotingServer; +import com.alibaba.rocketmq.remoting.netty.NettyRequestProcessor; +import com.alibaba.rocketmq.remoting.netty.NettyServerConfig; +import com.alibaba.rocketmq.remoting.netty.ResponseFuture; +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; + + +/** + * @author shijia.wxr + */ +public class NettyRPCTest { + public static RemotingClient createRemotingClient() { + NettyClientConfig config = new NettyClientConfig(); + RemotingClient client = new NettyRemotingClient(config); + client.start(); + return client; + } + + + public static RemotingServer createRemotingServer() throws InterruptedException { + NettyServerConfig config = new NettyServerConfig(); + RemotingServer remotingServer = new NettyRemotingServer(config); + remotingServer.registerProcessor(0, new NettyRequestProcessor() { + private int i = 0; + + + @Override + public RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand request) { + System.out.println("processRequest=" + request + " " + (i++)); + request.setRemark("hello, I am respponse " + ctx.channel().remoteAddress()); + return request; + } + }, Executors.newCachedThreadPool()); + remotingServer.start(); + return remotingServer; + } + + + @Test + public void test_RPC_Sync() throws InterruptedException, RemotingConnectException, + RemotingSendRequestException, RemotingTimeoutException { + RemotingServer server = createRemotingServer(); + RemotingClient client = createRemotingClient(); + + for (int i = 0; i < 100; i++) { + TestRequestHeader requestHeader = new TestRequestHeader(); + requestHeader.setCount(i); + requestHeader.setMessageTitle("HelloMessageTitle"); + RemotingCommand request = RemotingCommand.createRequestCommand(0, requestHeader); + RemotingCommand response = client.invokeSync("localhost:8888", request, 1000 * 3000); + System.out.println("invoke result = " + response); + assertTrue(response != null); + } + + client.shutdown(); + server.shutdown(); + System.out.println("-----------------------------------------------------------------"); + } + + + @Test + public void test_RPC_Oneway() throws InterruptedException, RemotingConnectException, + RemotingTimeoutException, RemotingTooMuchRequestException, RemotingSendRequestException { + RemotingServer server = createRemotingServer(); + RemotingClient client = createRemotingClient(); + + for (int i = 0; i < 100; i++) { + RemotingCommand request = RemotingCommand.createRequestCommand(0, null); + request.setRemark(String.valueOf(i)); + client.invokeOneway("localhost:8888", request, 1000 * 3); + } + + client.shutdown(); + server.shutdown(); + System.out.println("-----------------------------------------------------------------"); + } + + + @Test + public void test_RPC_Async() throws InterruptedException, RemotingConnectException, + RemotingTimeoutException, RemotingTooMuchRequestException, RemotingSendRequestException { + RemotingServer server = createRemotingServer(); + RemotingClient client = createRemotingClient(); + + for (int i = 0; i < 100; i++) { + RemotingCommand request = RemotingCommand.createRequestCommand(0, null); + request.setRemark(String.valueOf(i)); + client.invokeAsync("localhost:8888", request, 1000 * 3, new InvokeCallback() { + @Override + public void operationComplete(ResponseFuture responseFuture) { + System.out.println(responseFuture.getResponseCommand()); + } + }); + } + + Thread.sleep(1000 * 3); + + client.shutdown(); + server.shutdown(); + System.out.println("-----------------------------------------------------------------"); + } + + + @Test + public void test_server_call_client() throws InterruptedException, RemotingConnectException, + RemotingSendRequestException, RemotingTimeoutException { + final RemotingServer server = createRemotingServer(); + final RemotingClient client = createRemotingClient(); + + server.registerProcessor(0, new NettyRequestProcessor() { + @Override + public RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand request) { + try { + return server.invokeSync(ctx.channel(), request, 1000 * 10); + } + catch (InterruptedException e) { + e.printStackTrace(); + } + catch (RemotingSendRequestException e) { + e.printStackTrace(); + } + catch (RemotingTimeoutException e) { + e.printStackTrace(); + } + + return null; + } + }, Executors.newCachedThreadPool()); + + client.registerProcessor(0, new NettyRequestProcessor() { + @Override + public RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand request) { + System.out.println("client receive server request = " + request); + request.setRemark("client remark"); + return request; + } + }, Executors.newCachedThreadPool()); + + for (int i = 0; i < 3; i++) { + RemotingCommand request = RemotingCommand.createRequestCommand(0, null); + RemotingCommand response = client.invokeSync("localhost:8888", request, 1000 * 3); + System.out.println("invoke result = " + response); + assertTrue(response != null); + } + + client.shutdown(); + server.shutdown(); + System.out.println("-----------------------------------------------------------------"); + } + +} + + +class TestRequestHeader implements CommandCustomHeader { + @CFNullable + private Integer count; + + @CFNullable + private String messageTitle; + + + @Override + public void checkFields() throws RemotingCommandException { + } + + + public Integer getCount() { + return count; + } + + + public void setCount(Integer count) { + this.count = count; + } + + + public String getMessageTitle() { + return messageTitle; + } + + + public void setMessageTitle(String messageTitle) { + this.messageTitle = messageTitle; + } +} + + +class TestResponseHeader implements CommandCustomHeader { + @CFNullable + private Integer count; + + @CFNullable + private String messageTitle; + + + @Override + public void checkFields() throws RemotingCommandException { + + } + + + public Integer getCount() { + return count; + } + + + public void setCount(Integer count) { + this.count = count; + } + + + public String getMessageTitle() { + return messageTitle; + } + + + public void setMessageTitle(String messageTitle) { + this.messageTitle = messageTitle; + } +} diff --git a/rocketmq-remoting/src/test/java/com/alibaba/rocketmq/remoting/SyncInvokeTest.java b/rocketmq-remoting/src/test/java/com/alibaba/rocketmq/remoting/SyncInvokeTest.java new file mode 100644 index 000000000..1963c9d6a --- /dev/null +++ b/rocketmq-remoting/src/test/java/com/alibaba/rocketmq/remoting/SyncInvokeTest.java @@ -0,0 +1,39 @@ +/** + * $Id: SyncInvokeTest.java 1831 2013-05-16 01:39:51Z shijia.wxr $ + */ +package com.alibaba.rocketmq.remoting; + +import static org.junit.Assert.assertTrue; + +import org.junit.Test; + +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; + + +/** + * @author shijia.wxr + */ +public class SyncInvokeTest { + @Test + public void test_RPC_Sync() throws Exception { + RemotingServer server = NettyRPCTest.createRemotingServer(); + RemotingClient client = NettyRPCTest.createRemotingClient(); + + for (int i = 0; i < 100; i++) { + try { + RemotingCommand request = RemotingCommand.createRequestCommand(0, null); + RemotingCommand response = client.invokeSync("localhost:8888", request, 1000 * 3); + System.out.println(i + "\t" + "invoke result = " + response); + assertTrue(response != null); + } + catch (Exception e) { + e.printStackTrace(); + throw e; + } + } + + client.shutdown(); + server.shutdown(); + System.out.println("-----------------------------------------------------------------"); + } +} diff --git a/rocketmq-remoting/src/test/java/com/alibaba/rocketmq/subclass/TestSubClassAuto.java b/rocketmq-remoting/src/test/java/com/alibaba/rocketmq/subclass/TestSubClassAuto.java new file mode 100644 index 000000000..a6ef2e2dd --- /dev/null +++ b/rocketmq-remoting/src/test/java/com/alibaba/rocketmq/subclass/TestSubClassAuto.java @@ -0,0 +1,17 @@ +/** + * + */ +package com.alibaba.rocketmq.subclass; + +import org.junit.Test; + + +/** + * @author shijia.wxr + */ +public class TestSubClassAuto { + @Test + public void test_sub() { + + } +} diff --git a/rocketmq-srvutil/pom.xml b/rocketmq-srvutil/pom.xml new file mode 100644 index 000000000..2a521df96 --- /dev/null +++ b/rocketmq-srvutil/pom.xml @@ -0,0 +1,38 @@ + + + com.alibaba.rocketmq + rocketmq-all + 3.2.2 + + + 4.0.0 + jar + rocketmq-srvutil + rocketmq-srvutil ${project.version} + + + + + junit + junit + test + + + ${project.groupId} + rocketmq-remoting + + + ${project.groupId} + rocketmq-common + + + commons-cli + commons-cli + + + commons-io + commons-io + + + diff --git a/rocketmq-srvutil/src/main/java/com/alibaba/rocketmq/srvutil/ServerUtil.java b/rocketmq-srvutil/src/main/java/com/alibaba/rocketmq/srvutil/ServerUtil.java new file mode 100644 index 000000000..1ebffc113 --- /dev/null +++ b/rocketmq-srvutil/src/main/java/com/alibaba/rocketmq/srvutil/ServerUtil.java @@ -0,0 +1,95 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.srvutil; + +import java.util.Properties; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; + + +/** + * 只提供Server程序依赖，目的为了拆解客户端依赖，尽可能减少客户端的依赖 + * + * @author vive + * + */ +public class ServerUtil { + + public static Options buildCommandlineOptions(final Options options) { + Option opt = new Option("h", "help", false, "Print help"); + opt.setRequired(false); + options.addOption(opt); + + opt = + new Option("n", "namesrvAddr", true, + "Name server address list, eg: 192.168.0.1:9876;192.168.0.2:9876"); + opt.setRequired(false); + options.addOption(opt); + + return options; + } + + + public static CommandLine parseCmdLine(final String appName, String[] args, Options options, + CommandLineParser parser) { + HelpFormatter hf = new HelpFormatter(); + hf.setWidth(110); + CommandLine commandLine = null; + try { + commandLine = parser.parse(options, args); + if (commandLine.hasOption('h')) { + hf.printHelp(appName, options, true); + return null; + } + } + catch (ParseException e) { + hf.printHelp(appName, options, true); + } + + return commandLine; + } + + + public static void printCommandLineHelp(final String appName, final Options options) { + HelpFormatter hf = new HelpFormatter(); + hf.setWidth(110); + hf.printHelp(appName, options, true); + } + + + public static Properties commandLine2Properties(final CommandLine commandLine) { + Properties properties = new Properties(); + Option[] opts = commandLine.getOptions(); + + if (opts != null) { + for (Option opt : opts) { + String name = opt.getLongOpt(); + String value = commandLine.getOptionValue(name); + if (value != null) { + properties.setProperty(name, value); + } + } + } + + return properties; + } + +} diff --git a/rocketmq-srvutil/src/test/java/com/alibaba/rocketmq/srvutil/ServerUtilTest.java b/rocketmq-srvutil/src/test/java/com/alibaba/rocketmq/srvutil/ServerUtilTest.java new file mode 100644 index 000000000..eb8e5a678 --- /dev/null +++ b/rocketmq-srvutil/src/test/java/com/alibaba/rocketmq/srvutil/ServerUtilTest.java @@ -0,0 +1,11 @@ +package com.alibaba.rocketmq.srvutil; + +import org.junit.Test; + + +public class ServerUtilTest { + @Test + public void test1() { + // TODO Auto-generated constructor stub + } +} diff --git a/rocketmq-store/pom.xml b/rocketmq-store/pom.xml new file mode 100644 index 000000000..cc2599c47 --- /dev/null +++ b/rocketmq-store/pom.xml @@ -0,0 +1,25 @@ + + + com.alibaba.rocketmq + rocketmq-all + 3.2.2 + + + 4.0.0 + jar + rocketmq-store + rocketmq-store ${project.version} + + + + junit + junit + test + + + ${project.groupId} + rocketmq-common + + + diff --git a/rocketmq-store/sbin/map.sh b/rocketmq-store/sbin/map.sh new file mode 100644 index 000000000..a106d1447 --- /dev/null +++ b/rocketmq-store/sbin/map.sh @@ -0,0 +1 @@ +/opt/taobao/java/bin/java -server -Xms2g -Xmx2g -XX:NewSize=512M -XX:MaxNewSize=1g -XX:PermSize=256m -XX:MaxPermSize=256m -XX:SurvivorRatio=8 -XX:+UseConcMarkSweepGC -XX:+UseCMSCompactAtFullCollection -XX:CMSInitiatingOccupancyFraction=60 -XX:+CMSParallelRemarkEnabled -XX:+CMSClassUnloadingEnabled -XX:+UseCMSInitiatingOccupancyOnly -XX:SoftRefLRUPolicyMSPerMB=0 -XX:+DisableExplicitGC -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=9999 -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -cp ../target/metaq-store-2.0.0-SNAPSHOT.jar:/home/shijia.wxr/metaq/metaq/metaq-commons/target/metaq-commons-2.0.0-SNAPSHOT.jar:/home/shijia.wxr/.m2/repository/log4j/log4j/1.2.14/log4j-1.2.14.jar com.taobao.metaq.store.inspect.TestMmap 1024 16384 \ No newline at end of file diff --git a/rocketmq-store/sbin/put.sh b/rocketmq-store/sbin/put.sh new file mode 100644 index 000000000..129b5c3d6 --- /dev/null +++ b/rocketmq-store/sbin/put.sh @@ -0,0 +1 @@ +/opt/taobao/java/bin/java -server -Xms2g -Xmx2g -XX:NewSize=512M -XX:MaxNewSize=1g -XX:PermSize=256m -XX:MaxPermSize=256m -XX:SurvivorRatio=8 -XX:+UseConcMarkSweepGC -XX:+UseCMSCompactAtFullCollection -XX:CMSInitiatingOccupancyFraction=60 -XX:+CMSParallelRemarkEnabled -XX:+CMSClassUnloadingEnabled -XX:+UseCMSInitiatingOccupancyOnly -XX:SoftRefLRUPolicyMSPerMB=0 -XX:+DisableExplicitGC -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=9999 -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -cp ../target/metaq-store-2.0.0-SNAPSHOT.jar:/home/shijia.wxr/metaq/metaq/metaq-commons/target/metaq-commons-2.0.0-SNAPSHOT.jar:/home/shijia.wxr/.m2/repository/log4j/log4j/1.2.14/log4j-1.2.14.jar com.taobao.metaq.store.inspect.TestPutGetMessage -1 2048 512 true \ No newline at end of file diff --git a/rocketmq-store/sbin/response.sh b/rocketmq-store/sbin/response.sh new file mode 100644 index 000000000..b3657e8a7 --- /dev/null +++ b/rocketmq-store/sbin/response.sh @@ -0,0 +1 @@ +/opt/taobao/java/bin/java -server -Xms2g -Xmx2g -XX:NewSize=512M -XX:MaxNewSize=1g -XX:PermSize=256m -XX:MaxPermSize=256m -XX:SurvivorRatio=8 -XX:+UseConcMarkSweepGC -XX:+UseCMSCompactAtFullCollection -XX:CMSInitiatingOccupancyFraction=60 -XX:+CMSParallelRemarkEnabled -XX:+CMSClassUnloadingEnabled -XX:+UseCMSInitiatingOccupancyOnly -XX:SoftRefLRUPolicyMSPerMB=0 -XX:+DisableExplicitGC -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=9999 -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -cp ../target/metaq-store-2.0.0-SNAPSHOT.jar:/home/shijia.wxr/metaq/metaq/metaq-commons/target/metaq-commons-2.0.0-SNAPSHOT.jar:/home/shijia.wxr/.m2/repository/log4j/log4j/1.2.14/log4j-1.2.14.jar com.taobao.metaq.store.inspect.TestPutResponse -1 512 \ No newline at end of file diff --git a/rocketmq-store/sbin/showcpu.sh b/rocketmq-store/sbin/showcpu.sh new file mode 100644 index 000000000..cc5aff31e --- /dev/null +++ b/rocketmq-store/sbin/showcpu.sh @@ -0,0 +1,6 @@ +#!/bin/sh +while [ "1" == "1" ] +do + ps -eo "%p %C %c" |grep java + sleep 1 +done \ No newline at end of file diff --git a/rocketmq-store/sbin/showdirty.sh b/rocketmq-store/sbin/showdirty.sh new file mode 100644 index 000000000..1303c97b7 --- /dev/null +++ b/rocketmq-store/sbin/showdirty.sh @@ -0,0 +1,9 @@ +#!/bin/sh +while [ "1" == "1" ] +do + NOW=`date +%H%M%S` + output=dirty.`date +%Y%m%d` + DIRTY=`cat /proc/vmstat |grep nr_dirty` + echo $NOW $DIRTY >> $output + sleep 1 +done diff --git a/rocketmq-store/sbin/showdirty2.sh b/rocketmq-store/sbin/showdirty2.sh new file mode 100644 index 000000000..edc6e5c30 --- /dev/null +++ b/rocketmq-store/sbin/showdirty2.sh @@ -0,0 +1,9 @@ +#!/bin/sh +while [ "1" == "1" ] +do + dirty=`cat /proc/vmstat |grep nr_dirty|awk '{print $2}'` + total=`cat /proc/vmstat |grep nr_file_pages|awk '{print $2}'` + ratio=`echo "scale=4; $dirty/$total * 100" | bc` + echo "$dirty $total ${ratio}%" + sleep 1 +done \ No newline at end of file diff --git a/rocketmq-store/sbin/showiostat.sh b/rocketmq-store/sbin/showiostat.sh new file mode 100644 index 000000000..f49f86100 --- /dev/null +++ b/rocketmq-store/sbin/showiostat.sh @@ -0,0 +1,2 @@ +#!/bin/sh +iostat sdb1 -x 1 -t \ No newline at end of file diff --git a/rocketmq-store/sbin/showload.sh b/rocketmq-store/sbin/showload.sh new file mode 100644 index 000000000..d3c0d4581 --- /dev/null +++ b/rocketmq-store/sbin/showload.sh @@ -0,0 +1,6 @@ +#!/bin/sh +while [ "1" == "1" ] +do + uptime + sleep 1 +done \ No newline at end of file diff --git a/rocketmq-store/sbin/showmap.sh b/rocketmq-store/sbin/showmap.sh new file mode 100644 index 000000000..bffce277a --- /dev/null +++ b/rocketmq-store/sbin/showmap.sh @@ -0,0 +1,2 @@ +#!/bin/sh +ps ax | grep -i 'com.taobao.metaq' |grep java | grep -v grep | awk '{print $1}' | xargs pmap |grep metastore \ No newline at end of file diff --git a/rocketmq-store/sbin/showmaptotal.sh b/rocketmq-store/sbin/showmaptotal.sh new file mode 100644 index 000000000..b6e31741a --- /dev/null +++ b/rocketmq-store/sbin/showmaptotal.sh @@ -0,0 +1,6 @@ +#!/bin/sh +while [ "1" == "1" ] +do + ps ax | grep -i 'com.taobao.metaq' |grep java | grep -v grep | awk '{print $1}' | xargs pmap |grep metastore |wc -l + sleep 1 +done diff --git a/rocketmq-store/sbin/showvmstat.sh b/rocketmq-store/sbin/showvmstat.sh new file mode 100644 index 000000000..ac33bfafe --- /dev/null +++ b/rocketmq-store/sbin/showvmstat.sh @@ -0,0 +1,8 @@ +#!/bin/sh +while [ "1" == "1" ] +do + date + cat /proc/vmstat |egrep "nr_free_pages|nr_anon_pages|nr_file_pages|nr_dirty|nr_writeback|pgpgout|pgsteal_normal|pgscan_kswapd_normal|pgscan_direct_normal|kswapd_steal" + echo + sleep 1 +done diff --git a/rocketmq-store/sbin/test.sh b/rocketmq-store/sbin/test.sh new file mode 100644 index 000000000..571ed8911 --- /dev/null +++ b/rocketmq-store/sbin/test.sh @@ -0,0 +1,6 @@ +#!/bin/sh +nohup sh put.sh > put.txt & +nohup sh showload.sh > load.txt & +nohup sh showiostat.sh > iostat.txt & +nohup sh showmaptotal.sh > map.txt & +nohup sh showcpu.sh > cpu.txt & \ No newline at end of file diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/AllocateMapedFileService.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/AllocateMapedFileService.java new file mode 100644 index 000000000..e5b8eba1a --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/AllocateMapedFileService.java @@ -0,0 +1,236 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store; + +import java.io.IOException; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.PriorityBlockingQueue; +import java.util.concurrent.TimeUnit; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.rocketmq.common.ServiceThread; +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.constant.LoggerName; + + +/** + * 预分配MapedFile服务 + * + * @author shijia.wxr + * @since 2013-7-21 + */ +public class AllocateMapedFileService extends ServiceThread { + private static final Logger log = LoggerFactory.getLogger(LoggerName.StoreLoggerName); + private static int WaitTimeOut = 1000 * 5; + private ConcurrentHashMap requestTable = + new ConcurrentHashMap(); + private PriorityBlockingQueue requestQueue = + new PriorityBlockingQueue(); + private volatile boolean hasException = false; + + + public MapedFile putRequestAndReturnMapedFile(String nextFilePath, String nextNextFilePath, int fileSize) { + AllocateRequest nextReq = new AllocateRequest(nextFilePath, fileSize); + AllocateRequest nextNextReq = new AllocateRequest(nextNextFilePath, fileSize); + boolean nextPutOK = (this.requestTable.putIfAbsent(nextFilePath, nextReq) == null); + boolean nextNextPutOK = (this.requestTable.putIfAbsent(nextNextFilePath, nextNextReq) == null); + + if (nextPutOK) { + boolean offerOK = this.requestQueue.offer(nextReq); + if (!offerOK) { + log.warn("add a request to preallocate queue failed"); + } + } + + if (nextNextPutOK) { + boolean offerOK = this.requestQueue.offer(nextNextReq); + if (!offerOK) { + log.warn("add a request to preallocate queue failed"); + } + } + + if (hasException) { + log.warn(this.getServiceName() + " service has exception. so return null"); + return null; + } + + AllocateRequest result = this.requestTable.get(nextFilePath); + try { + if (result != null) { + boolean waitOK = result.getCountDownLatch().await(WaitTimeOut, TimeUnit.MILLISECONDS); + if (!waitOK) { + log.warn("create mmap timeout " + result.getFilePath() + " " + result.getFileSize()); + } + this.requestTable.remove(nextFilePath); + return result.getMapedFile(); + } + else { + log.error("find preallocate mmap failed, this never happen"); + } + } + catch (InterruptedException e) { + log.warn(this.getServiceName() + " service has exception. ", e); + } + + return null; + } + + + @Override + public String getServiceName() { + return AllocateMapedFileService.class.getSimpleName(); + } + + + public void shutdown() { + this.stoped = true; + this.thread.interrupt(); + + try { + this.thread.join(this.getJointime()); + } + catch (InterruptedException e) { + e.printStackTrace(); + } + + for (AllocateRequest req : this.requestTable.values()) { + if (req.mapedFile != null) { + log.info("delete pre allocated maped file, {}", req.mapedFile.getFileName()); + req.mapedFile.destroy(1000); + } + } + } + + + public void run() { + log.info(this.getServiceName() + " service started"); + + while (!this.isStoped() && this.mmapOperation()) + ; + + log.info(this.getServiceName() + " service end"); + } + + + /** + * 只有被外部线程中断，才会返回false + */ + private boolean mmapOperation() { + AllocateRequest req = null; + try { + req = this.requestQueue.take(); + if (null == this.requestTable.get(req.getFilePath())) { + log.warn("this mmap request expired, maybe cause timeout " + req.getFilePath() + " " + + req.getFileSize()); + return true; + } + + if (req.getMapedFile() == null) { + long beginTime = System.currentTimeMillis(); + MapedFile mapedFile = new MapedFile(req.getFilePath(), req.getFileSize()); + long eclipseTime = UtilAll.computeEclipseTimeMilliseconds(beginTime); + // 记录大于10ms的 + if (eclipseTime > 10) { + int queueSize = this.requestQueue.size(); + log.warn("create mapedFile spent time(ms) " + eclipseTime + " queue size " + queueSize + + " " + req.getFilePath() + " " + req.getFileSize()); + } + + req.setMapedFile(mapedFile); + this.hasException = false; + } + } + catch (InterruptedException e) { + log.warn(this.getServiceName() + " service has exception, maybe by shutdown"); + this.hasException = true; + return false; + } + catch (IOException e) { + log.warn(this.getServiceName() + " service has exception. ", e); + this.hasException = true; + } + finally { + if (req != null) + req.getCountDownLatch().countDown(); + } + return true; + } + + class AllocateRequest implements Comparable { + // 文件全路径 + private String filePath; + // 文件大小 + private int fileSize; + // 计数器 + private CountDownLatch countDownLatch = new CountDownLatch(1); + // MapedFile + private volatile MapedFile mapedFile = null; + + + public AllocateRequest(String filePath, int fileSize) { + this.filePath = filePath; + this.fileSize = fileSize; + } + + + public String getFilePath() { + return filePath; + } + + + public void setFilePath(String filePath) { + this.filePath = filePath; + } + + + public int getFileSize() { + return fileSize; + } + + + public void setFileSize(int fileSize) { + this.fileSize = fileSize; + } + + + public CountDownLatch getCountDownLatch() { + return countDownLatch; + } + + + public void setCountDownLatch(CountDownLatch countDownLatch) { + this.countDownLatch = countDownLatch; + } + + + public MapedFile getMapedFile() { + return mapedFile; + } + + + public void setMapedFile(MapedFile mapedFile) { + this.mapedFile = mapedFile; + } + + + public int compareTo(AllocateRequest other) { + return this.fileSize < other.fileSize ? 1 : this.fileSize > other.fileSize ? -1 : 0; + } + } +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/AppendMessageCallback.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/AppendMessageCallback.java new file mode 100644 index 000000000..90b6a604a --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/AppendMessageCallback.java @@ -0,0 +1,42 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store; + +import java.nio.ByteBuffer; + + +/** + * 写入消息的回调接口 + * + * @author shijia.wxr + * @since 2013-7-21 + */ +public interface AppendMessageCallback { + + /** + * 序列化消息后，写入MapedByteBuffer + * + * @param byteBuffer + * 要写入的target + * @param maxBlank + * 要写入的target最大空白区 + * @param msg + * 要写入的message + * @return 写入多少字节 + */ + public AppendMessageResult doAppend(final long fileFromOffset, final ByteBuffer byteBuffer, + final int maxBlank, final Object msg); +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/AppendMessageResult.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/AppendMessageResult.java new file mode 100644 index 000000000..2623574b5 --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/AppendMessageResult.java @@ -0,0 +1,127 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store; + +/** + * 向物理队列写入消息返回结果 + * + * @author shijia.wxr + * @since 2013-7-21 + */ +public class AppendMessageResult { + // 返回码 + private AppendMessageStatus status; + // 从哪里开始写入 + private long wroteOffset; + // 写入字节数 + private int wroteBytes; + // 消息ID + private String msgId; + // 消息存储时间 + private long storeTimestamp; + // 写入逻辑队列的offset（递进1） + private long logicsOffset; + + + public AppendMessageResult(AppendMessageStatus status) { + this(status, 0, 0, "", 0, 0); + } + + + public AppendMessageResult(AppendMessageStatus status, long wroteOffset, int wroteBytes, String msgId, + long storeTimestamp, long logicsOffset) { + this.status = status; + this.wroteOffset = wroteOffset; + this.wroteBytes = wroteBytes; + this.msgId = msgId; + this.storeTimestamp = storeTimestamp; + this.logicsOffset = logicsOffset; + } + + + public boolean isOk() { + return this.status == AppendMessageStatus.PUT_OK; + } + + + public AppendMessageStatus getStatus() { + return status; + } + + + public void setStatus(AppendMessageStatus status) { + this.status = status; + } + + + public long getWroteOffset() { + return wroteOffset; + } + + + public void setWroteOffset(long wroteOffset) { + this.wroteOffset = wroteOffset; + } + + + public int getWroteBytes() { + return wroteBytes; + } + + + public void setWroteBytes(int wroteBytes) { + this.wroteBytes = wroteBytes; + } + + + public String getMsgId() { + return msgId; + } + + + public void setMsgId(String msgId) { + this.msgId = msgId; + } + + + public long getStoreTimestamp() { + return storeTimestamp; + } + + + public void setStoreTimestamp(long storeTimestamp) { + this.storeTimestamp = storeTimestamp; + } + + + public long getLogicsOffset() { + return logicsOffset; + } + + + public void setLogicsOffset(long logicsOffset) { + this.logicsOffset = logicsOffset; + } + + + @Override + public String toString() { + return "AppendMessageResult [status=" + status + ", wroteOffset=" + wroteOffset + ", wroteBytes=" + + wroteBytes + ", msgId=" + msgId + ", storeTimestamp=" + storeTimestamp + ", logicsOffset=" + + logicsOffset + "]"; + } + +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/AppendMessageStatus.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/AppendMessageStatus.java new file mode 100644 index 000000000..2af1e8f2c --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/AppendMessageStatus.java @@ -0,0 +1,33 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store; + +/** + * 向物理队列写消息返回结果码 + * + * @author shijia.wxr + * @since 2013-7-21 + */ +public enum AppendMessageStatus { + // 成功追加消息 + PUT_OK, + // 走到文件末尾 + END_OF_FILE, + // 消息大小超限 + MESSAGE_SIZE_EXCEEDED, + // 未知错误 + UNKNOWN_ERROR, +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/CommitLog.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/CommitLog.java new file mode 100644 index 000000000..2717233a0 --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/CommitLog.java @@ -0,0 +1,1179 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store; + +import java.nio.BufferUnderflowException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.rocketmq.common.ServiceThread; +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.common.message.MessageAccessor; +import com.alibaba.rocketmq.common.message.MessageConst; +import com.alibaba.rocketmq.common.message.MessageDecoder; +import com.alibaba.rocketmq.common.message.MessageExt; +import com.alibaba.rocketmq.common.sysflag.MessageSysFlag; +import com.alibaba.rocketmq.store.config.BrokerRole; +import com.alibaba.rocketmq.store.config.FlushDiskType; +import com.alibaba.rocketmq.store.ha.HAService; +import com.alibaba.rocketmq.store.schedule.ScheduleMessageService; + + +/** + * CommitLog实现 + * + * @author shijia.wxr + * @since 2013-7-21 + */ +public class CommitLog { + private static final Logger log = LoggerFactory.getLogger(LoggerName.StoreLoggerName); + // 每个消息对应的MAGIC CODE daa320a7 + public final static int MessageMagicCode = 0xAABBCCDD ^ 1880681586 + 8; + // 文件末尾空洞对应的MAGIC CODE cbd43194 + private final static int BlankMagicCode = 0xBBCCDDEE ^ 1880681586 + 8; + // 存储消息的队列 + private final MapedFileQueue mapedFileQueue; + // 存储顶层对象 + private final DefaultMessageStore defaultMessageStore; + // CommitLog刷盘服务 + private final FlushCommitLogService flushCommitLogService; + // 存储消息时的回调接口 + private final AppendMessageCallback appendMessageCallback; + // 用来保存每个ConsumeQueue的当前最大Offset信息 + private HashMap topicQueueTable = new HashMap( + 1024); + + + /** + * 构造函数 + */ + public CommitLog(final DefaultMessageStore defaultMessageStore) { + this.mapedFileQueue = + new MapedFileQueue(defaultMessageStore.getMessageStoreConfig().getStorePathCommitLog(), + defaultMessageStore.getMessageStoreConfig().getMapedFileSizeCommitLog(), + defaultMessageStore.getAllocateMapedFileService()); + this.defaultMessageStore = defaultMessageStore; + + if (FlushDiskType.SYNC_FLUSH == defaultMessageStore.getMessageStoreConfig().getFlushDiskType()) { + this.flushCommitLogService = new GroupCommitService(); + } + else { + this.flushCommitLogService = new FlushRealTimeService(); + } + + this.appendMessageCallback = + new DefaultAppendMessageCallback(defaultMessageStore.getMessageStoreConfig() + .getMaxMessageSize()); + } + + + public boolean load() { + boolean result = this.mapedFileQueue.load(); + log.info("load commit log " + (result ? "OK" : "Failed")); + return result; + } + + + public void start() { + this.flushCommitLogService.start(); + } + + + public void shutdown() { + this.flushCommitLogService.shutdown(); + } + + + public long getMinOffset() { + MapedFile mapedFile = this.mapedFileQueue.getFirstMapedFileOnLock(); + if (mapedFile != null) { + if (mapedFile.isAvailable()) { + return mapedFile.getFileFromOffset(); + } + else { + return this.rollNextFile(mapedFile.getFileFromOffset()); + } + } + + return -1; + } + + + public long rollNextFile(final long offset) { + int mapedFileSize = this.defaultMessageStore.getMessageStoreConfig().getMapedFileSizeCommitLog(); + return (offset + mapedFileSize - offset % mapedFileSize); + } + + + public long getMaxOffset() { + return this.mapedFileQueue.getMaxOffset(); + } + + + public int deleteExpiredFile(// + final long expiredTime, // + final int deleteFilesInterval, // + final long intervalForcibly,// + final boolean cleanImmediately// + ) { + return this.mapedFileQueue.deleteExpiredFileByTime(expiredTime, deleteFilesInterval, + intervalForcibly, cleanImmediately); + } + + + /** + * 读取CommitLog数据，数据复制时使用 + */ + public SelectMapedBufferResult getData(final long offset) { + return this.getData(offset, (0 == offset ? true : false)); + } + + + public SelectMapedBufferResult getData(final long offset, final boolean returnFirstOnNotFound) { + int mapedFileSize = this.defaultMessageStore.getMessageStoreConfig().getMapedFileSizeCommitLog(); + MapedFile mapedFile = this.mapedFileQueue.findMapedFileByOffset(offset, returnFirstOnNotFound); + if (mapedFile != null) { + int pos = (int) (offset % mapedFileSize); + SelectMapedBufferResult result = mapedFile.selectMapedBuffer(pos); + return result; + } + + return null; + } + + + /** + * 正常退出时，数据恢复，所有内存数据都已经刷盘 + */ + public void recoverNormally() { + boolean checkCRCOnRecover = this.defaultMessageStore.getMessageStoreConfig().isCheckCRCOnRecover(); + final List mapedFiles = this.mapedFileQueue.getMapedFiles(); + if (!mapedFiles.isEmpty()) { + // 从倒数第三个文件开始恢复 + int index = mapedFiles.size() - 3; + if (index < 0) + index = 0; + + MapedFile mapedFile = mapedFiles.get(index); + ByteBuffer byteBuffer = mapedFile.sliceByteBuffer(); + long processOffset = mapedFile.getFileFromOffset(); + long mapedFileOffset = 0; + while (true) { + DispatchRequest dispatchRequest = + this.checkMessageAndReturnSize(byteBuffer, checkCRCOnRecover); + int size = dispatchRequest.getMsgSize(); + // 正常数据 + if (size > 0) { + mapedFileOffset += size; + } + // 文件中间读到错误 + else if (size == -1) { + log.info("recover physics file end, " + mapedFile.getFileName()); + break; + } + // 走到文件末尾，切换至下一个文件 + // 由于返回0代表是遇到了最后的空洞，这个可以不计入truncate offset中 + else if (size == 0) { + index++; + if (index >= mapedFiles.size()) { + // 当前条件分支不可能发生 + log.info("recover last 3 physics file over, last maped file " + + mapedFile.getFileName()); + break; + } + else { + mapedFile = mapedFiles.get(index); + byteBuffer = mapedFile.sliceByteBuffer(); + processOffset = mapedFile.getFileFromOffset(); + mapedFileOffset = 0; + log.info("recover next physics file, " + mapedFile.getFileName()); + } + } + } + + processOffset += mapedFileOffset; + this.mapedFileQueue.setCommittedWhere(processOffset); + this.mapedFileQueue.truncateDirtyFiles(processOffset); + } + } + + + public DispatchRequest checkMessageAndReturnSize(java.nio.ByteBuffer byteBuffer, final boolean checkCRC) { + return this.checkMessageAndReturnSize(byteBuffer, checkCRC, true); + } + + + /** + * 服务端使用 检查消息并返回消息大小 + * + * @return 0 表示走到文件末尾 >0 正常消息 -1 消息校验失败 + */ + public DispatchRequest checkMessageAndReturnSize(java.nio.ByteBuffer byteBuffer, final boolean checkCRC, + final boolean readBody) { + try { + java.nio.ByteBuffer byteBufferMessage = + ((DefaultAppendMessageCallback) this.appendMessageCallback).getMsgStoreItemMemory(); + byte[] bytesContent = byteBufferMessage.array(); + + // 1 TOTALSIZE + int totalSize = byteBuffer.getInt(); + + // 2 MAGICCODE + int magicCode = byteBuffer.getInt(); + switch (magicCode) { + case MessageMagicCode: + break; + case BlankMagicCode: + return new DispatchRequest(0); + default: + log.warn("found a illegal magic code 0x" + Integer.toHexString(magicCode)); + return new DispatchRequest(-1); + } + + // 3 BODYCRC + int bodyCRC = byteBuffer.getInt(); + + // 4 QUEUEID + int queueId = byteBuffer.getInt(); + + // 5 FLAG + int flag = byteBuffer.getInt(); + flag = flag + 0; + + // 6 QUEUEOFFSET + long queueOffset = byteBuffer.getLong(); + + // 7 PHYSICALOFFSET + long physicOffset = byteBuffer.getLong(); + + // 8 SYSFLAG + int sysFlag = byteBuffer.getInt(); + + // 9 BORNTIMESTAMP + long bornTimeStamp = byteBuffer.getLong(); + bornTimeStamp = bornTimeStamp + 0; + + // 10 BORNHOST（IP+PORT） + byteBuffer.get(bytesContent, 0, 8); + + // 11 STORETIMESTAMP + long storeTimestamp = byteBuffer.getLong(); + + // 12 STOREHOST（IP+PORT） + byteBuffer.get(bytesContent, 0, 8); + + // 13 RECONSUMETIMES + int reconsumeTimes = byteBuffer.getInt(); + + // 14 Prepared Transaction Offset + long preparedTransactionOffset = byteBuffer.getLong(); + + // 15 BODY + int bodyLen = byteBuffer.getInt(); + if (bodyLen > 0) { + if (readBody) { + byteBuffer.get(bytesContent, 0, bodyLen); + + // 校验CRC + if (checkCRC) { + int crc = UtilAll.crc32(bytesContent, 0, bodyLen); + if (crc != bodyCRC) { + log.warn("CRC check failed " + crc + " " + bodyCRC); + return new DispatchRequest(-1); + } + } + } + else { + byteBuffer.position(byteBuffer.position() + bodyLen); + } + } + + // 16 TOPIC + byte topicLen = byteBuffer.get(); + byteBuffer.get(bytesContent, 0, topicLen); + String topic = new String(bytesContent, 0, topicLen); + + long tagsCode = 0; + String keys = ""; + + // 17 properties + short propertiesLength = byteBuffer.getShort(); + if (propertiesLength > 0) { + byteBuffer.get(bytesContent, 0, propertiesLength); + String properties = new String(bytesContent, 0, propertiesLength); + Map propertiesMap = MessageDecoder.string2messageProperties(properties); + + keys = propertiesMap.get(MessageConst.PROPERTY_KEYS); + String tags = propertiesMap.get(MessageConst.PROPERTY_TAGS); + if (tags != null && tags.length() > 0) { + tagsCode = + MessageExtBrokerInner.tagsString2tagsCode( + MessageExt.parseTopicFilterType(sysFlag), tags); + } + + // 定时消息处理 + { + String t = propertiesMap.get(MessageConst.PROPERTY_DELAY_TIME_LEVEL); + if (ScheduleMessageService.SCHEDULE_TOPIC.equals(topic) && t != null) { + int delayLevel = Integer.parseInt(t); + + if (delayLevel > this.defaultMessageStore.getScheduleMessageService() + .getMaxDelayLevel()) { + delayLevel = + this.defaultMessageStore.getScheduleMessageService().getMaxDelayLevel(); + } + + if (delayLevel > 0) { + tagsCode = + this.defaultMessageStore.getScheduleMessageService() + .computeDeliverTimestamp(delayLevel, storeTimestamp); + } + } + } + } + + return new DispatchRequest(// + topic,// 1 + queueId,// 2 + physicOffset,// 3 + totalSize,// 4 + tagsCode,// 5 + storeTimestamp,// 6 + queueOffset,// 7 + keys,// 8 + sysFlag,// 9 + preparedTransactionOffset// 10 + ); + } + catch (BufferUnderflowException e) { + byteBuffer.position(byteBuffer.limit()); + } + catch (Exception e) { + byteBuffer.position(byteBuffer.limit()); + } + + return new DispatchRequest(-1); + } + + + public void recoverAbnormally() { + // 根据最小时间戳来恢复 + boolean checkCRCOnRecover = this.defaultMessageStore.getMessageStoreConfig().isCheckCRCOnRecover(); + final List mapedFiles = this.mapedFileQueue.getMapedFiles(); + if (!mapedFiles.isEmpty()) { + // 寻找从哪个文件开始恢复 + int index = mapedFiles.size() - 1; + MapedFile mapedFile = null; + for (; index >= 0; index--) { + mapedFile = mapedFiles.get(index); + if (this.isMapedFileMatchedRecover(mapedFile)) { + log.info("recover from this maped file " + mapedFile.getFileName()); + break; + } + } + + if (index < 0) { + index = 0; + mapedFile = mapedFiles.get(index); + } + + ByteBuffer byteBuffer = mapedFile.sliceByteBuffer(); + long processOffset = mapedFile.getFileFromOffset(); + long mapedFileOffset = 0; + while (true) { + DispatchRequest dispatchRequest = + this.checkMessageAndReturnSize(byteBuffer, checkCRCOnRecover); + int size = dispatchRequest.getMsgSize(); + // 正常数据 + if (size > 0) { + mapedFileOffset += size; + this.defaultMessageStore.putDispatchRequest(dispatchRequest); + } + // 文件中间读到错误 + else if (size == -1) { + log.info("recover physics file end, " + mapedFile.getFileName()); + break; + } + // 走到文件末尾，切换至下一个文件 + // 由于返回0代表是遇到了最后的空洞，这个可以不计入truncate offset中 + else if (size == 0) { + index++; + if (index >= mapedFiles.size()) { + // 当前条件分支正常情况下不应该发生 + log.info("recover physics file over, last maped file " + mapedFile.getFileName()); + break; + } + else { + mapedFile = mapedFiles.get(index); + byteBuffer = mapedFile.sliceByteBuffer(); + processOffset = mapedFile.getFileFromOffset(); + mapedFileOffset = 0; + log.info("recover next physics file, " + mapedFile.getFileName()); + } + } + } + + processOffset += mapedFileOffset; + this.mapedFileQueue.setCommittedWhere(processOffset); + this.mapedFileQueue.truncateDirtyFiles(processOffset); + + // 清除ConsumeQueue的多余数据 + this.defaultMessageStore.truncateDirtyLogicFiles(processOffset); + } + // 物理文件都被删除情况下 + else { + this.mapedFileQueue.setCommittedWhere(0); + this.defaultMessageStore.destroyLogics(); + } + } + + + private boolean isMapedFileMatchedRecover(final MapedFile mapedFile) { + ByteBuffer byteBuffer = mapedFile.sliceByteBuffer(); + + int magicCode = byteBuffer.getInt(MessageDecoder.MessageMagicCodePostion); + if (magicCode != MessageMagicCode) { + return false; + } + + long storeTimestamp = byteBuffer.getLong(MessageDecoder.MessageStoreTimestampPostion); + if (0 == storeTimestamp) { + return false; + } + + if (this.defaultMessageStore.getMessageStoreConfig().isMessageIndexEnable()// + && this.defaultMessageStore.getMessageStoreConfig().isMessageIndexSafe()) { + if (storeTimestamp <= this.defaultMessageStore.getStoreCheckpoint().getMinTimestampIndex()) { + log.info("find check timestamp, {} {}", // + storeTimestamp,// + UtilAll.timeMillisToHumanString(storeTimestamp)); + return true; + } + } + else { + if (storeTimestamp <= this.defaultMessageStore.getStoreCheckpoint().getMinTimestamp()) { + log.info("find check timestamp, {} {}", // + storeTimestamp,// + UtilAll.timeMillisToHumanString(storeTimestamp)); + return true; + } + } + + return false; + } + + + public PutMessageResult putMessage(final MessageExtBrokerInner msg) { + // 设置存储时间 + msg.setStoreTimestamp(System.currentTimeMillis()); + // 设置消息体BODY CRC（考虑在客户端设置最合适） + msg.setBodyCRC(UtilAll.crc32(msg.getBody())); + // 返回结果 + AppendMessageResult result = null; + + StoreStatsService storeStatsService = this.defaultMessageStore.getStoreStatsService(); + + String topic = msg.getTopic(); + int queueId = msg.getQueueId(); + long tagsCode = msg.getTagsCode(); + + final int tranType = MessageSysFlag.getTransactionValue(msg.getSysFlag()); + if (tranType == MessageSysFlag.TransactionNotType// + || tranType == MessageSysFlag.TransactionCommitType) { + // 延时投递 + if (msg.getDelayTimeLevel() > 0) { + if (msg.getDelayTimeLevel() > this.defaultMessageStore.getScheduleMessageService() + .getMaxDelayLevel()) { + msg.setDelayTimeLevel(this.defaultMessageStore.getScheduleMessageService() + .getMaxDelayLevel()); + } + + topic = ScheduleMessageService.SCHEDULE_TOPIC; + queueId = ScheduleMessageService.delayLevel2QueueId(msg.getDelayTimeLevel()); + tagsCode = + this.defaultMessageStore.getScheduleMessageService().computeDeliverTimestamp( + msg.getDelayTimeLevel(), msg.getStoreTimestamp()); + + /** + * 备份真实的topic，queueId + */ + MessageAccessor.putProperty(msg, MessageConst.PROPERTY_REAL_TOPIC, msg.getTopic()); + MessageAccessor.putProperty(msg, MessageConst.PROPERTY_REAL_QUEUE_ID, + String.valueOf(msg.getQueueId())); + msg.setPropertiesString(MessageDecoder.messageProperties2String(msg.getProperties())); + + msg.setTopic(topic); + msg.setQueueId(queueId); + } + } + + // 写文件要加锁 + long eclipseTimeInLock = 0; + synchronized (this) { + long beginLockTimestamp = this.defaultMessageStore.getSystemClock().now(); + + // 这里设置存储时间戳，才能保证全局有序 + msg.setStoreTimestamp(beginLockTimestamp); + + // 尝试写入 + MapedFile mapedFile = this.mapedFileQueue.getLastMapedFile(); + if (null == mapedFile) { + log.error("create maped file1 error, topic: " + msg.getTopic() + " clientAddr: " + + msg.getBornHostString()); + return new PutMessageResult(PutMessageStatus.CREATE_MAPEDFILE_FAILED, null); + } + result = mapedFile.appendMessage(msg, this.appendMessageCallback); + switch (result.getStatus()) { + // 成功追加消息 + case PUT_OK: + break; + // 走到文件末尾 + case END_OF_FILE: + // 创建新文件，重新写消息 + mapedFile = this.mapedFileQueue.getLastMapedFile(); + if (null == mapedFile) { + // XXX: warn and notify me + log.error("create maped file2 error, topic: " + msg.getTopic() + " clientAddr: " + + msg.getBornHostString()); + return new PutMessageResult(PutMessageStatus.CREATE_MAPEDFILE_FAILED, result); + } + result = mapedFile.appendMessage(msg, this.appendMessageCallback); + break; + // 消息大小超限 + case MESSAGE_SIZE_EXCEEDED: + return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, result); + // 未知错误 + case UNKNOWN_ERROR: + return new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, result); + default: + return new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, result); + } + + DispatchRequest dispatchRequest = new DispatchRequest(// + topic,// 1 + queueId,// 2 + result.getWroteOffset(),// 3 + result.getWroteBytes(),// 4 + tagsCode,// 5 + msg.getStoreTimestamp(),// 6 + result.getLogicsOffset(),// 7 + msg.getKeys(),// 8 + /** + * 事务部分 + */ + msg.getSysFlag(),// 9 + msg.getPreparedTransactionOffset());// 10 + + this.defaultMessageStore.putDispatchRequest(dispatchRequest); + + eclipseTimeInLock = this.defaultMessageStore.getSystemClock().now() - beginLockTimestamp; + } // end of synchronized + + if (eclipseTimeInLock > 1000) { + // XXX: warn and notify me + log.warn("putMessage in lock eclipse time(ms) " + eclipseTimeInLock); + } + + // 返回结果 + PutMessageResult putMessageResult = new PutMessageResult(PutMessageStatus.PUT_OK, result); + + // 统计消息SIZE + storeStatsService.getSinglePutMessageTopicSizeTotal(topic).addAndGet(result.getWroteBytes()); + + GroupCommitRequest request = null; + + // 同步刷盘 + if (FlushDiskType.SYNC_FLUSH == this.defaultMessageStore.getMessageStoreConfig().getFlushDiskType()) { + GroupCommitService service = (GroupCommitService) this.flushCommitLogService; + if (msg.isWaitStoreMsgOK()) { + request = new GroupCommitRequest(result.getWroteOffset() + result.getWroteBytes()); + service.putRequest(request); + boolean flushOK = + request.waitForFlush(this.defaultMessageStore.getMessageStoreConfig() + .getSyncFlushTimeout()); + if (!flushOK) { + log.error("do groupcommit, wait for flush failed, topic: " + msg.getTopic() + " tags: " + + msg.getTags() + " client address: " + msg.getBornHostString()); + putMessageResult.setPutMessageStatus(PutMessageStatus.FLUSH_DISK_TIMEOUT); + } + } + else { + service.wakeup(); + } + } + // 异步刷盘 + else { + this.flushCommitLogService.wakeup(); + } + + // 同步双写 + if (BrokerRole.SYNC_MASTER == this.defaultMessageStore.getMessageStoreConfig().getBrokerRole()) { + HAService service = this.defaultMessageStore.getHaService(); + if (msg.isWaitStoreMsgOK()) { + // 判断是否要等待 + if (service.isSlaveOK(result.getWroteOffset() + result.getWroteBytes())) { + if (null == request) { + request = new GroupCommitRequest(result.getWroteOffset() + result.getWroteBytes()); + } + service.putRequest(request); + + service.getWaitNotifyObject().wakeupAll(); + + boolean flushOK = + // TODO 此处参数与刷盘公用是否合适 + request.waitForFlush(this.defaultMessageStore.getMessageStoreConfig() + .getSyncFlushTimeout()); + if (!flushOK) { + log.error("do sync transfer other node, wait return, but failed, topic: " + + msg.getTopic() + " tags: " + msg.getTags() + " client address: " + + msg.getBornHostString()); + putMessageResult.setPutMessageStatus(PutMessageStatus.FLUSH_SLAVE_TIMEOUT); + } + } + // Slave异常 + else { + // 告诉发送方，Slave异常 + putMessageResult.setPutMessageStatus(PutMessageStatus.SLAVE_NOT_AVAILABLE); + } + } + } + + // 向发送方返回结果 + return putMessageResult; + } + + + /** + * 根据offset获取特定消息的存储时间 如果出错，则返回-1 + */ + public long pickupStoretimestamp(final long offset, final int size) { + if (offset > this.getMinOffset()) { + SelectMapedBufferResult result = this.getMessage(offset, size); + if (null != result) { + try { + return result.getByteBuffer().getLong(MessageDecoder.MessageStoreTimestampPostion); + } + finally { + result.release(); + } + } + } + + return -1; + } + + + /** + * 读取消息 + */ + public SelectMapedBufferResult getMessage(final long offset, final int size) { + int mapedFileSize = this.defaultMessageStore.getMessageStoreConfig().getMapedFileSizeCommitLog(); + MapedFile mapedFile = this.mapedFileQueue.findMapedFileByOffset(offset, (0 == offset ? true : false)); + if (mapedFile != null) { + int pos = (int) (offset % mapedFileSize); + SelectMapedBufferResult result = mapedFile.selectMapedBuffer(pos, size); + return result; + } + + return null; + } + + + public HashMap getTopicQueueTable() { + return topicQueueTable; + } + + + public void setTopicQueueTable(HashMap topicQueueTable) { + this.topicQueueTable = topicQueueTable; + } + + + public void destroy() { + this.mapedFileQueue.destroy(); + } + + + public boolean appendData(long startOffset, byte[] data) { + // 写文件要加锁 + synchronized (this) { + // 尝试写入 + MapedFile mapedFile = this.mapedFileQueue.getLastMapedFile(startOffset); + if (null == mapedFile) { + log.error("appendData getLastMapedFile error " + startOffset); + return false; + } + + return mapedFile.appendMessage(data); + } + } + + + public boolean retryDeleteFirstFile(final long intervalForcibly) { + return this.mapedFileQueue.retryDeleteFirstFile(intervalForcibly); + } + + abstract class FlushCommitLogService extends ServiceThread { + } + + /** + * 异步实时刷盘服务 + */ + class FlushRealTimeService extends FlushCommitLogService { + private static final int RetryTimesOver = 3; + private long lastFlushTimestamp = 0; + private long printTimes = 0; + + + public void run() { + CommitLog.log.info(this.getServiceName() + " service started"); + + while (!this.isStoped()) { + boolean flushCommitLogTimed = + CommitLog.this.defaultMessageStore.getMessageStoreConfig().isFlushCommitLogTimed(); + + int interval = + CommitLog.this.defaultMessageStore.getMessageStoreConfig() + .getFlushIntervalCommitLog(); + int flushPhysicQueueLeastPages = + CommitLog.this.defaultMessageStore.getMessageStoreConfig() + .getFlushCommitLogLeastPages(); + + int flushPhysicQueueThoroughInterval = + CommitLog.this.defaultMessageStore.getMessageStoreConfig() + .getFlushCommitLogThoroughInterval(); + + boolean printFlushProgress = false; + + // 定时刷盘，定时打印刷盘进度 + long currentTimeMillis = System.currentTimeMillis(); + if (currentTimeMillis >= (this.lastFlushTimestamp + flushPhysicQueueThoroughInterval)) { + this.lastFlushTimestamp = currentTimeMillis; + flushPhysicQueueLeastPages = 0; + printFlushProgress = ((printTimes++ % 10) == 0); + } + + try { + // 定时刷盘 + if (flushCommitLogTimed) { + Thread.sleep(interval); + } + // 实时刷盘 + else { + this.waitForRunning(interval); + } + + if (printFlushProgress) { + this.printFlushProgress(); + } + + CommitLog.this.mapedFileQueue.commit(flushPhysicQueueLeastPages); + long storeTimestamp = CommitLog.this.mapedFileQueue.getStoreTimestamp(); + if (storeTimestamp > 0) { + CommitLog.this.defaultMessageStore.getStoreCheckpoint().setPhysicMsgTimestamp( + storeTimestamp); + } + } + catch (Exception e) { + CommitLog.log.warn(this.getServiceName() + " service has exception. ", e); + this.printFlushProgress(); + } + } + + // 正常shutdown时，要保证全部刷盘才退出 + boolean result = false; + for (int i = 0; i < RetryTimesOver && !result; i++) { + result = CommitLog.this.mapedFileQueue.commit(0); + CommitLog.log.info(this.getServiceName() + " service shutdown, retry " + (i + 1) + " times " + + (result ? "OK" : "Not OK")); + } + + this.printFlushProgress(); + + CommitLog.log.info(this.getServiceName() + " service end"); + } + + + @Override + public String getServiceName() { + return FlushCommitLogService.class.getSimpleName(); + } + + + private void printFlushProgress() { + CommitLog.log.info("how much disk fall behind memory, " + + CommitLog.this.mapedFileQueue.howMuchFallBehind()); + } + + + @Override + public long getJointime() { + // 由于CommitLog数据量较大，所以回收时间要更长 + return 1000 * 60 * 5; + } + } + + public class GroupCommitRequest { + // 当前消息对应的下一个Offset + private final long nextOffset; + // 异步通知对象 + private final CountDownLatch countDownLatch = new CountDownLatch(1); + // 刷盘是否成功 + private volatile boolean flushOK = false; + + + public GroupCommitRequest(long nextOffset) { + this.nextOffset = nextOffset; + } + + + public long getNextOffset() { + return nextOffset; + } + + + public void wakeupCustomer(final boolean flushOK) { + this.flushOK = flushOK; + this.countDownLatch.countDown(); + } + + + public boolean waitForFlush(long timeout) { + try { + boolean result = this.countDownLatch.await(timeout, TimeUnit.MILLISECONDS); + return result || this.flushOK; + } + catch (InterruptedException e) { + e.printStackTrace(); + return false; + } + } + } + + /** + * GroupCommit Service + */ + class GroupCommitService extends FlushCommitLogService { + private volatile List requestsWrite = new ArrayList(); + private volatile List requestsRead = new ArrayList(); + + + public void putRequest(final GroupCommitRequest request) { + synchronized (this) { + this.requestsWrite.add(request); + if (!this.hasNotified) { + this.hasNotified = true; + this.notify(); + } + } + } + + + private void swapRequests() { + List tmp = this.requestsWrite; + this.requestsWrite = this.requestsRead; + this.requestsRead = tmp; + } + + + private void doCommit() { + if (!this.requestsRead.isEmpty()) { + for (GroupCommitRequest req : this.requestsRead) { + // 消息有可能在下一个文件，所以最多刷盘2次 + boolean flushOK = false; + for (int i = 0; (i < 2) && !flushOK; i++) { + flushOK = (CommitLog.this.mapedFileQueue.getCommittedWhere() >= req.getNextOffset()); + + if (!flushOK) { + CommitLog.this.mapedFileQueue.commit(0); + } + } + + req.wakeupCustomer(flushOK); + } + + long storeTimestamp = CommitLog.this.mapedFileQueue.getStoreTimestamp(); + if (storeTimestamp > 0) { + CommitLog.this.defaultMessageStore.getStoreCheckpoint().setPhysicMsgTimestamp( + storeTimestamp); + } + + this.requestsRead.clear(); + } + else { + // 由于个别消息设置为不同步刷盘，所以会走到此流程 + CommitLog.this.mapedFileQueue.commit(0); + } + } + + + public void run() { + CommitLog.log.info(this.getServiceName() + " service started"); + + while (!this.isStoped()) { + try { + this.waitForRunning(0); + this.doCommit(); + } + catch (Exception e) { + CommitLog.log.warn(this.getServiceName() + " service has exception. ", e); + } + } + + // 在正常shutdown情况下，等待请求到来，然后再刷盘 + try { + Thread.sleep(10); + } + catch (InterruptedException e) { + CommitLog.log.warn("GroupCommitService Exception, ", e); + } + + synchronized (this) { + this.swapRequests(); + } + + this.doCommit(); + + CommitLog.log.info(this.getServiceName() + " service end"); + } + + + @Override + protected void onWaitEnd() { + this.swapRequests(); + } + + + @Override + public String getServiceName() { + return GroupCommitService.class.getSimpleName(); + } + + + @Override + public long getJointime() { + // 由于CommitLog数据量较大，所以回收时间要更长 + return 1000 * 60 * 5; + } + } + + class DefaultAppendMessageCallback implements AppendMessageCallback { + // 文件末尾空洞最小定长 + private static final int END_FILE_MIN_BLANK_LENGTH = 4 + 4; + // 存储消息ID + private final ByteBuffer msgIdMemory; + // 存储消息内容 + private final ByteBuffer msgStoreItemMemory; + // 消息的最大长度 + private final int maxMessageSize; + + + DefaultAppendMessageCallback(final int size) { + this.msgIdMemory = ByteBuffer.allocate(MessageDecoder.MSG_ID_LENGTH); + this.msgStoreItemMemory = ByteBuffer.allocate(size + END_FILE_MIN_BLANK_LENGTH); + this.maxMessageSize = size; + } + + + public ByteBuffer getMsgStoreItemMemory() { + return msgStoreItemMemory; + } + + + public AppendMessageResult doAppend(final long fileFromOffset, final ByteBuffer byteBuffer, + final int maxBlank, final Object msg) { + /** + * 生成消息ID STORETIMESTAMP + STOREHOSTADDRESS + OFFSET
+ */ + MessageExtBrokerInner msgInner = (MessageExtBrokerInner) msg; + // PHY OFFSET + long wroteOffset = fileFromOffset + byteBuffer.position(); + String msgId = + MessageDecoder.createMessageId(this.msgIdMemory, msgInner.getStoreHostBytes(), + wroteOffset); + + /** + * 记录ConsumeQueue信息 + */ + String key = msgInner.getTopic() + "-" + msgInner.getQueueId(); + Long queueOffset = CommitLog.this.topicQueueTable.get(key); + if (null == queueOffset) { + queueOffset = 0L; + CommitLog.this.topicQueueTable.put(key, queueOffset); + } + + /** + * 事务消息需要特殊处理 + */ + final int tranType = MessageSysFlag.getTransactionValue(msgInner.getSysFlag()); + switch (tranType) { + // Prepared和Rollback都是不可以消费的消息，不会进入消费队列 + case MessageSysFlag.TransactionPreparedType: + case MessageSysFlag.TransactionRollbackType: + queueOffset = 0L; + break; + case MessageSysFlag.TransactionNotType: + case MessageSysFlag.TransactionCommitType: + default: + break; + } + + /** + * 序列化消息 + */ + final byte[] propertiesData = + msgInner.getPropertiesString() == null ? null : msgInner.getPropertiesString().getBytes(); + final int propertiesLength = propertiesData == null ? 0 : propertiesData.length; + + final byte[] topicData = msgInner.getTopic().getBytes(); + final int topicLength = topicData == null ? 0 : topicData.length; + + final int bodyLength = msgInner.getBody() == null ? 0 : msgInner.getBody().length; + + final int msgLen = 4 // 1 TOTALSIZE + + 4 // 2 MAGICCODE + + 4 // 3 BODYCRC + + 4 // 4 QUEUEID + + 4 // 5 FLAG + + 8 // 6 QUEUEOFFSET + + 8 // 7 PHYSICALOFFSET + + 4 // 8 SYSFLAG + + 8 // 9 BORNTIMESTAMP + + 8 // 10 BORNHOST + + 8 // 11 STORETIMESTAMP + + 8 // 12 STOREHOSTADDRESS + + 4 // 13 RECONSUMETIMES + + 8 // 14 Prepared Transaction Offset + + 4 + bodyLength // 14 BODY + + 1 + topicLength // 15 TOPIC + + 2 + propertiesLength // 16 propertiesLength + + 0; + + // 消息超过设定的最大值 + if (msgLen > this.maxMessageSize) { + CommitLog.log.warn("message size exceeded, msg total size: " + msgLen + ", msg body size: " + + bodyLength + ", maxMessageSize: " + this.maxMessageSize); + return new AppendMessageResult(AppendMessageStatus.MESSAGE_SIZE_EXCEEDED); + } + + // 判断是否有足够空余空间 + if ((msgLen + END_FILE_MIN_BLANK_LENGTH) > maxBlank) { + this.resetMsgStoreItemMemory(maxBlank); + // 1 TOTALSIZE + this.msgStoreItemMemory.putInt(maxBlank); + // 2 MAGICCODE + this.msgStoreItemMemory.putInt(CommitLog.BlankMagicCode); + // 3 剩余空间可能是任何值 + // + + // 此处长度特意设置为maxBlank + byteBuffer.put(this.msgStoreItemMemory.array(), 0, maxBlank); + return new AppendMessageResult(AppendMessageStatus.END_OF_FILE, wroteOffset, maxBlank, msgId, + msgInner.getStoreTimestamp(), queueOffset); + } + + // 初始化存储空间 + this.resetMsgStoreItemMemory(msgLen); + // 1 TOTALSIZE + this.msgStoreItemMemory.putInt(msgLen); + // 2 MAGICCODE + this.msgStoreItemMemory.putInt(CommitLog.MessageMagicCode); + // 3 BODYCRC + this.msgStoreItemMemory.putInt(msgInner.getBodyCRC()); + // 4 QUEUEID + this.msgStoreItemMemory.putInt(msgInner.getQueueId()); + // 5 FLAG + this.msgStoreItemMemory.putInt(msgInner.getFlag()); + // 6 QUEUEOFFSET + this.msgStoreItemMemory.putLong(queueOffset); + // 7 PHYSICALOFFSET + this.msgStoreItemMemory.putLong(fileFromOffset + byteBuffer.position()); + // 8 SYSFLAG + this.msgStoreItemMemory.putInt(msgInner.getSysFlag()); + // 9 BORNTIMESTAMP + this.msgStoreItemMemory.putLong(msgInner.getBornTimestamp()); + // 10 BORNHOST + this.msgStoreItemMemory.put(msgInner.getBornHostBytes()); + // 11 STORETIMESTAMP + this.msgStoreItemMemory.putLong(msgInner.getStoreTimestamp()); + // 12 STOREHOSTADDRESS + this.msgStoreItemMemory.put(msgInner.getStoreHostBytes()); + // 13 RECONSUMETIMES + this.msgStoreItemMemory.putInt(msgInner.getReconsumeTimes()); + // 14 Prepared Transaction Offset + this.msgStoreItemMemory.putLong(msgInner.getPreparedTransactionOffset()); + // 15 BODY + this.msgStoreItemMemory.putInt(bodyLength); + if (bodyLength > 0) + this.msgStoreItemMemory.put(msgInner.getBody()); + // 16 TOPIC + this.msgStoreItemMemory.put((byte) topicLength); + this.msgStoreItemMemory.put(topicData); + // 17 PROPERTIES + this.msgStoreItemMemory.putShort((short) propertiesLength); + if (propertiesLength > 0) + this.msgStoreItemMemory.put(propertiesData); + + // 向队列缓冲区写入消息 + byteBuffer.put(this.msgStoreItemMemory.array(), 0, msgLen); + + AppendMessageResult result = + new AppendMessageResult(AppendMessageStatus.PUT_OK, wroteOffset, msgLen, msgId, + msgInner.getStoreTimestamp(), queueOffset); + + switch (tranType) { + case MessageSysFlag.TransactionPreparedType: + case MessageSysFlag.TransactionRollbackType: + break; + case MessageSysFlag.TransactionNotType: + case MessageSysFlag.TransactionCommitType: + // 更新下一次的ConsumeQueue信息 + CommitLog.this.topicQueueTable.put(key, ++queueOffset); + break; + default: + break; + } + + // 返回结果 + return result; + } + + + private void resetMsgStoreItemMemory(final int length) { + this.msgStoreItemMemory.flip(); + this.msgStoreItemMemory.limit(length); + } + } + + + public void removeQueurFromTopicQueueTable(final String topic, final int queueId) { + String key = topic + "-" + queueId; + synchronized (this) { + this.topicQueueTable.remove(key); + } + + log.info("removeQueurFromTopicQueueTable OK Topic: {} QueueId: {}", topic, queueId); + } +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/ConsumeQueue.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/ConsumeQueue.java new file mode 100644 index 000000000..f48df5758 --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/ConsumeQueue.java @@ -0,0 +1,577 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store; + +import java.io.File; +import java.nio.ByteBuffer; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.rocketmq.common.constant.LoggerName; + + +/** + * 消费队列实现 + * + * @author shijia.wxr + * @since 2013-7-21 + */ +public class ConsumeQueue { + // 存储单元大小 + public static final int CQStoreUnitSize = 20; + private static final Logger log = LoggerFactory.getLogger(LoggerName.StoreLoggerName); + private static final Logger logError = LoggerFactory.getLogger(LoggerName.StoreErrorLoggerName); + // 存储顶层对象 + private final DefaultMessageStore defaultMessageStore; + // 存储消息索引的队列 + private final MapedFileQueue mapedFileQueue; + // Topic + private final String topic; + // queueId + private final int queueId; + // 写索引时用到的ByteBuffer + private final ByteBuffer byteBufferIndex; + // 配置 + private final String storePath; + private final int mapedFileSize; + // 最后一个消息对应的物理Offset + private long maxPhysicOffset = -1; + // 逻辑队列的最小Offset，删除物理文件时，计算出来的最小Offset + // 实际使用需要除以 StoreUnitSize + private volatile long minLogicOffset = 0; + + + public ConsumeQueue(// + final String topic,// + final int queueId,// + final String storePath,// + final int mapedFileSize,// + final DefaultMessageStore defaultMessageStore) { + this.storePath = storePath; + this.mapedFileSize = mapedFileSize; + this.defaultMessageStore = defaultMessageStore; + + this.topic = topic; + this.queueId = queueId; + + String queueDir = this.storePath// + + File.separator + topic// + + File.separator + queueId;// + + this.mapedFileQueue = new MapedFileQueue(queueDir, mapedFileSize, null); + + this.byteBufferIndex = ByteBuffer.allocate(CQStoreUnitSize); + } + + + public boolean load() { + boolean result = this.mapedFileQueue.load(); + log.info("load consume queue " + this.topic + "-" + this.queueId + " " + (result ? "OK" : "Failed")); + return result; + } + + + public void recover() { + final List mapedFiles = this.mapedFileQueue.getMapedFiles(); + if (!mapedFiles.isEmpty()) { + // 从倒数第三个文件开始恢复 + int index = mapedFiles.size() - 3; + if (index < 0) + index = 0; + + int mapedFileSizeLogics = this.mapedFileSize; + MapedFile mapedFile = mapedFiles.get(index); + ByteBuffer byteBuffer = mapedFile.sliceByteBuffer(); + long processOffset = mapedFile.getFileFromOffset(); + long mapedFileOffset = 0; + while (true) { + for (int i = 0; i < mapedFileSizeLogics; i += CQStoreUnitSize) { + long offset = byteBuffer.getLong(); + int size = byteBuffer.getInt(); + long tagsCode = byteBuffer.getLong(); + + // 说明当前存储单元有效 + // TODO 这样判断有效是否合理？ + if (offset >= 0 && size > 0) { + mapedFileOffset = i + CQStoreUnitSize; + this.maxPhysicOffset = offset; + } + else { + log.info("recover current consume queue file over, " + mapedFile.getFileName() + " " + + offset + " " + size + " " + tagsCode); + break; + } + } + + // 走到文件末尾，切换至下一个文件 + if (mapedFileOffset == mapedFileSizeLogics) { + index++; + if (index >= mapedFiles.size()) { + // 当前条件分支不可能发生 + log.info("recover last consume queue file over, last maped file " + + mapedFile.getFileName()); + break; + } + else { + mapedFile = mapedFiles.get(index); + byteBuffer = mapedFile.sliceByteBuffer(); + processOffset = mapedFile.getFileFromOffset(); + mapedFileOffset = 0; + log.info("recover next consume queue file, " + mapedFile.getFileName()); + } + } + else { + log.info("recover current consume queue queue over " + mapedFile.getFileName() + " " + + (processOffset + mapedFileOffset)); + break; + } + } + + processOffset += mapedFileOffset; + this.mapedFileQueue.truncateDirtyFiles(processOffset); + } + } + + + /** + * 二分查找查找消息发送时间最接近timestamp逻辑队列的offset + */ + public long getOffsetInQueueByTime(final long timestamp) { + MapedFile mapedFile = this.mapedFileQueue.getMapedFileByTime(timestamp); + if (mapedFile != null) { + long offset = 0; + // low:第一个索引信息的起始位置 + // minLogicOffset有设置值则从 + // minLogicOffset-mapedFile.getFileFromOffset()位置开始才是有效值 + int low = + minLogicOffset > mapedFile.getFileFromOffset() ? (int) (minLogicOffset - mapedFile + .getFileFromOffset()) : 0; + + // high:最后一个索引信息的起始位置 + int high = 0; + int midOffset = -1, targetOffset = -1, leftOffset = -1, rightOffset = -1; + long leftIndexValue = -1L, rightIndexValue = -1L; + + // 取出该mapedFile里面所有的映射空间(没有映射的空间并不会返回,不会返回文件空洞) + SelectMapedBufferResult sbr = mapedFile.selectMapedBuffer(0); + if (null != sbr) { + ByteBuffer byteBuffer = sbr.getByteBuffer(); + high = byteBuffer.limit() - CQStoreUnitSize; + try { + while (high >= low) { + midOffset = (low + high) / (2 * CQStoreUnitSize) * CQStoreUnitSize; + byteBuffer.position(midOffset); + long phyOffset = byteBuffer.getLong(); + int size = byteBuffer.getInt(); + + // 比较时间, 折半 + long storeTime = + this.defaultMessageStore.getCommitLog().pickupStoretimestamp(phyOffset, size); + if (storeTime < 0) { + // 没有从物理文件找到消息，此时直接返回0 + return 0; + } + else if (storeTime == timestamp) { + targetOffset = midOffset; + break; + } + else if (storeTime > timestamp) { + high = midOffset - CQStoreUnitSize; + rightOffset = midOffset; + rightIndexValue = storeTime; + } + else { + low = midOffset + CQStoreUnitSize; + leftOffset = midOffset; + leftIndexValue = storeTime; + } + } + + if (targetOffset != -1) { + // 查询的时间正好是消息索引记录写入的时间 + offset = targetOffset; + } + else { + if (leftIndexValue == -1) { + // timestamp 时间小于该MapedFile中第一条记录记录的时间 + offset = rightOffset; + } + else if (rightIndexValue == -1) { + // timestamp 时间大于该MapedFile中最后一条记录记录的时间 + offset = leftOffset; + } + else { + // 取最接近timestamp的offset + offset = + Math.abs(timestamp - leftIndexValue) > Math.abs(timestamp + - rightIndexValue) ? rightOffset : leftOffset; + } + } + + return (mapedFile.getFileFromOffset() + offset) / CQStoreUnitSize; + } + finally { + sbr.release(); + } + } + } + + // 映射文件被标记为不可用时返回0 + return 0; + } + + + /** + * 根据物理Offset删除无效逻辑文件 + */ + public void truncateDirtyLogicFiles(long phyOffet) { + // 逻辑队列每个文件大小 + int logicFileSize = this.mapedFileSize; + + // 先改变逻辑队列存储的物理Offset + this.maxPhysicOffset = phyOffet - 1; + + while (true) { + MapedFile mapedFile = this.mapedFileQueue.getLastMapedFile2(); + if (mapedFile != null) { + ByteBuffer byteBuffer = mapedFile.sliceByteBuffer(); + // 先将Offset清空 + mapedFile.setWrotePostion(0); + mapedFile.setCommittedPosition(0); + + for (int i = 0; i < logicFileSize; i += CQStoreUnitSize) { + long offset = byteBuffer.getLong(); + int size = byteBuffer.getInt(); + byteBuffer.getLong(); + + // 逻辑文件起始单元 + if (0 == i) { + if (offset >= phyOffet) { + this.mapedFileQueue.deleteLastMapedFile(); + break; + } + else { + int pos = i + CQStoreUnitSize; + mapedFile.setWrotePostion(pos); + mapedFile.setCommittedPosition(pos); + this.maxPhysicOffset = offset; + } + } + // 逻辑文件中间单元 + else { + // 说明当前存储单元有效 + if (offset >= 0 && size > 0) { + // 如果逻辑队列存储的最大物理offset大于物理队列最大offset，则返回 + if (offset >= phyOffet) { + return; + } + + int pos = i + CQStoreUnitSize; + mapedFile.setWrotePostion(pos); + mapedFile.setCommittedPosition(pos); + this.maxPhysicOffset = offset; + + // 如果最后一个MapedFile扫描完，则返回 + if (pos == logicFileSize) { + return; + } + } + else { + return; + } + } + } + } + else { + break; + } + } + } + + + /** + * 返回最后一条消息对应物理队列的Next Offset + */ + public long getLastOffset() { + // 物理队列Offset + long lastOffset = -1; + // 逻辑队列每个文件大小 + int logicFileSize = this.mapedFileSize; + + MapedFile mapedFile = this.mapedFileQueue.getLastMapedFile2(); + if (mapedFile != null) { + // 找到写入位置对应的索引项的起始位置 + int position = mapedFile.getWrotePostion() - CQStoreUnitSize; + if (position < 0) + position = 0; + + ByteBuffer byteBuffer = mapedFile.sliceByteBuffer(); + byteBuffer.position(position); + for (int i = 0; i < logicFileSize; i += CQStoreUnitSize) { + long offset = byteBuffer.getLong(); + int size = byteBuffer.getInt(); + byteBuffer.getLong(); + + // 说明当前存储单元有效 + if (offset >= 0 && size > 0) { + lastOffset = offset + size; + } + else { + break; + } + } + } + + return lastOffset; + } + + + public boolean commit(final int flushLeastPages) { + return this.mapedFileQueue.commit(flushLeastPages); + } + + + public int deleteExpiredFile(long offset) { + int cnt = this.mapedFileQueue.deleteExpiredFileByOffset(offset, CQStoreUnitSize); + // 无论是否删除文件，都需要纠正下最小值，因为有可能物理文件删除了， + // 但是逻辑文件一个也删除不了 + this.correctMinOffset(offset); + return cnt; + } + + + /** + * 逻辑队列的最小Offset要比传入的物理最小phyMinOffset大 + */ + public void correctMinOffset(long phyMinOffset) { + MapedFile mapedFile = this.mapedFileQueue.getFirstMapedFileOnLock(); + if (mapedFile != null) { + SelectMapedBufferResult result = mapedFile.selectMapedBuffer(0); + if (result != null) { + try { + // 有消息存在 + for (int i = 0; i < result.getSize(); i += ConsumeQueue.CQStoreUnitSize) { + long offsetPy = result.getByteBuffer().getLong(); + result.getByteBuffer().getInt(); + result.getByteBuffer().getLong(); + + if (offsetPy >= phyMinOffset) { + this.minLogicOffset = result.getMapedFile().getFileFromOffset() + i; + log.info("compute logics min offset: " + this.getMinOffsetInQuque() + ", topic: " + + this.topic + ", queueId: " + this.queueId); + break; + } + } + } + catch (Exception e) { + e.printStackTrace(); + } + finally { + result.release(); + } + } + } + } + + + public long getMinOffsetInQuque() { + return this.minLogicOffset / CQStoreUnitSize; + } + + + public void putMessagePostionInfoWrapper(long offset, int size, long tagsCode, long storeTimestamp, + long logicOffset) { + final int MaxRetries = 5; + boolean canWrite = this.defaultMessageStore.getRunningFlags().isWriteable(); + for (int i = 0; i < MaxRetries && canWrite; i++) { + boolean result = this.putMessagePostionInfo(offset, size, tagsCode, logicOffset); + if (result) { + this.defaultMessageStore.getStoreCheckpoint().setLogicsMsgTimestamp(storeTimestamp); + return; + } + // 只有一种情况会失败，创建新的MapedFile时报错或者超时 + else { + // XXX: warn and notify me + log.warn("[BUG]put commit log postion info to " + topic + ":" + queueId + " " + offset + + " failed, retry " + i + " times"); + + try { + Thread.sleep(1000); + } + catch (InterruptedException e) { + log.warn("", e); + } + } + } + + // XXX: warn and notify me + log.error("[BUG]consume queue can not write, {} {}", this.topic, this.queueId); + this.defaultMessageStore.getRunningFlags().makeLogicsQueueError(); + } + + + /** + * 存储一个20字节的信息，putMessagePostionInfo只有一个线程调用，所以不需要加锁 + * + * @param offset + * 消息对应的CommitLog offset + * @param size + * 消息在CommitLog存储的大小 + * @param tagsCode + * tags 计算出来的长整数 + * @return 是否成功 + */ + private boolean putMessagePostionInfo(final long offset, final int size, final long tagsCode, + final long cqOffset) { + // 在数据恢复时会走到这个流程 + if (offset <= this.maxPhysicOffset) { + return true; + } + + this.byteBufferIndex.flip(); + this.byteBufferIndex.limit(CQStoreUnitSize); + this.byteBufferIndex.putLong(offset); + this.byteBufferIndex.putInt(size); + this.byteBufferIndex.putLong(tagsCode); + + final long expectLogicOffset = cqOffset * CQStoreUnitSize; + + MapedFile mapedFile = this.mapedFileQueue.getLastMapedFile(expectLogicOffset); + if (mapedFile != null) { + // 纠正MapedFile逻辑队列索引顺序 + if (mapedFile.isFirstCreateInQueue() && cqOffset != 0 && mapedFile.getWrotePostion() == 0) { + this.minLogicOffset = expectLogicOffset; + this.fillPreBlank(mapedFile, expectLogicOffset); + log.info("fill pre blank space " + mapedFile.getFileName() + " " + expectLogicOffset + " " + + mapedFile.getWrotePostion()); + } + + if (cqOffset != 0) { + long currentLogicOffset = mapedFile.getWrotePostion() + mapedFile.getFileFromOffset(); + if (expectLogicOffset != currentLogicOffset) { + // XXX: warn and notify me + logError + .warn( + "[BUG]logic queue order maybe wrong, expectLogicOffset: {} currentLogicOffset: {} Topic: {} QID: {} Diff: {}",// + expectLogicOffset, // + currentLogicOffset,// + this.topic,// + this.queueId,// + expectLogicOffset - currentLogicOffset// + ); + } + } + + // 记录物理队列最大offset + this.maxPhysicOffset = offset; + return mapedFile.appendMessage(this.byteBufferIndex.array()); + } + + return false; + } + + + private void fillPreBlank(final MapedFile mapedFile, final long untilWhere) { + ByteBuffer byteBuffer = ByteBuffer.allocate(CQStoreUnitSize); + byteBuffer.putLong(0L); + byteBuffer.putInt(Integer.MAX_VALUE); + byteBuffer.putLong(0L); + + int until = (int) (untilWhere % this.mapedFileQueue.getMapedFileSize()); + for (int i = 0; i < until; i += CQStoreUnitSize) { + mapedFile.appendMessage(byteBuffer.array()); + } + } + + + /** + * 返回Index Buffer + * + * @param startIndex + * 起始偏移量索引 + */ + public SelectMapedBufferResult getIndexBuffer(final long startIndex) { + int mapedFileSize = this.mapedFileSize; + long offset = startIndex * CQStoreUnitSize; + if (offset >= this.getMinLogicOffset()) { + MapedFile mapedFile = this.mapedFileQueue.findMapedFileByOffset(offset); + if (mapedFile != null) { + SelectMapedBufferResult result = mapedFile.selectMapedBuffer((int) (offset % mapedFileSize)); + return result; + } + } + return null; + } + + + public long rollNextFile(final long index) { + int mapedFileSize = this.mapedFileSize; + int totalUnitsInFile = mapedFileSize / CQStoreUnitSize; + return (index + totalUnitsInFile - index % totalUnitsInFile); + } + + + public String getTopic() { + return topic; + } + + + public int getQueueId() { + return queueId; + } + + + public long getMaxPhysicOffset() { + return maxPhysicOffset; + } + + + public void setMaxPhysicOffset(long maxPhysicOffset) { + this.maxPhysicOffset = maxPhysicOffset; + } + + + public void destroy() { + this.maxPhysicOffset = -1; + this.minLogicOffset = 0; + this.mapedFileQueue.destroy(); + } + + + public long getMinLogicOffset() { + return minLogicOffset; + } + + + public void setMinLogicOffset(long minLogicOffset) { + this.minLogicOffset = minLogicOffset; + } + + + /** + * 获取当前队列中的消息总数 + */ + public long getMessageTotalInQueue() { + return this.getMaxOffsetInQuque() - this.getMinOffsetInQuque(); + } + + + public long getMaxOffsetInQuque() { + return this.mapedFileQueue.getMaxOffset() / CQStoreUnitSize; + } +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/DefaultMessageFilter.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/DefaultMessageFilter.java new file mode 100644 index 000000000..6742c1843 --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/DefaultMessageFilter.java @@ -0,0 +1,45 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store; + +import com.alibaba.rocketmq.common.protocol.heartbeat.SubscriptionData; + + +/** + * 消息过滤规则实现 + * + * @author shijia.wxr + * @since 2013-7-21 + */ +public class DefaultMessageFilter implements MessageFilter { + + @Override + public boolean isMessageMatched(SubscriptionData subscriptionData, long tagsCode) { + if (null == subscriptionData) { + return true; + } + + if (subscriptionData.isClassFilterMode()) + return true; + + if (subscriptionData.getSubString().equals(SubscriptionData.SUB_ALL)) { + return true; + } + + return subscriptionData.getCodeSet().contains((int) tagsCode); + } + +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/DefaultMessageStore.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/DefaultMessageStore.java new file mode 100644 index 000000000..fe4b08ffb --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/DefaultMessageStore.java @@ -0,0 +1,1930 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store; + +import com.alibaba.rocketmq.common.ServiceThread; +import com.alibaba.rocketmq.common.SystemClock; +import com.alibaba.rocketmq.common.ThreadFactoryImpl; +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.common.message.MessageConst; +import com.alibaba.rocketmq.common.message.MessageDecoder; +import com.alibaba.rocketmq.common.message.MessageExt; +import com.alibaba.rocketmq.common.protocol.heartbeat.SubscriptionData; +import com.alibaba.rocketmq.common.running.RunningStats; +import com.alibaba.rocketmq.common.sysflag.MessageSysFlag; +import com.alibaba.rocketmq.store.config.BrokerRole; +import com.alibaba.rocketmq.store.config.MessageStoreConfig; +import com.alibaba.rocketmq.store.config.StorePathConfigHelper; +import com.alibaba.rocketmq.store.ha.HAService; +import com.alibaba.rocketmq.store.index.IndexService; +import com.alibaba.rocketmq.store.index.QueryOffsetResult; +import com.alibaba.rocketmq.store.schedule.ScheduleMessageService; +import com.alibaba.rocketmq.store.stats.BrokerStatsManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.net.SocketAddress; +import java.nio.ByteBuffer; +import java.util.*; +import java.util.Map.Entry; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +import static com.alibaba.rocketmq.store.config.BrokerRole.SLAVE; + + +/** + * 存储层默认实现 + * + * @author shijia.wxr + * @since 2013-7-21 + */ +public class DefaultMessageStore implements MessageStore { + private static final Logger log = LoggerFactory.getLogger(LoggerName.StoreLoggerName); + // 消息过滤 + private final MessageFilter messageFilter = new DefaultMessageFilter(); + // 存储配置 + private final MessageStoreConfig messageStoreConfig; + // CommitLog + private final CommitLog commitLog; + // ConsumeQueue集合 + private final ConcurrentHashMap> consumeQueueTable; + // 逻辑队列刷盘服务 + private final FlushConsumeQueueService flushConsumeQueueService; + // 清理物理文件服务 + private final CleanCommitLogService cleanCommitLogService; + // 清理逻辑文件服务 + private final CleanConsumeQueueService cleanConsumeQueueService; + // 分发消息索引服务 + private final DispatchMessageService dispatchMessageService; + // 消息索引服务 + private final IndexService indexService; + // 预分配MapedFile对象服务 + private final AllocateMapedFileService allocateMapedFileService; + // 从物理队列解析消息重新发送到逻辑队列 + private final ReputMessageService reputMessageService; + // HA服务 + private final HAService haService; + // 定时服务 + private final ScheduleMessageService scheduleMessageService; + // 运行时数据统计 + private final StoreStatsService storeStatsService; + // 运行过程标志位 + private final RunningFlags runningFlags = new RunningFlags(); + // 优化获取时间性能，精度1ms + private final SystemClock systemClock = new SystemClock(1); + // 存储服务是否启动 + private volatile boolean shutdown = true; + // 存储检查点 + private StoreCheckpoint storeCheckpoint; + // 权限控制后，打印间隔次数 + private AtomicLong printTimes = new AtomicLong(0); + // 存储层的定时线程 + private final ScheduledExecutorService scheduledExecutorService = Executors + .newSingleThreadScheduledExecutor(new ThreadFactoryImpl("StoreScheduledThread")); + private final BrokerStatsManager brokerStatsManager; + + + public DefaultMessageStore(final MessageStoreConfig messageStoreConfig, + final BrokerStatsManager brokerStatsManager) throws IOException { + this.messageStoreConfig = messageStoreConfig; + this.brokerStatsManager = brokerStatsManager; + this.allocateMapedFileService = new AllocateMapedFileService(); + this.commitLog = new CommitLog(this); + this.consumeQueueTable = + new ConcurrentHashMap>( + 32); + + this.flushConsumeQueueService = new FlushConsumeQueueService(); + this.cleanCommitLogService = new CleanCommitLogService(); + this.cleanConsumeQueueService = new CleanConsumeQueueService(); + this.dispatchMessageService = + new DispatchMessageService(this.messageStoreConfig.getPutMsgIndexHightWater()); + this.storeStatsService = new StoreStatsService(); + this.indexService = new IndexService(this); + this.haService = new HAService(this); + + switch (this.messageStoreConfig.getBrokerRole()) { + case SLAVE: + this.reputMessageService = new ReputMessageService(); + // reputMessageService依赖scheduleMessageService做定时消息的恢复，确保储备数据一致 + this.scheduleMessageService = new ScheduleMessageService(this); + break; + case ASYNC_MASTER: + case SYNC_MASTER: + this.reputMessageService = null; + this.scheduleMessageService = new ScheduleMessageService(this); + break; + default: + this.reputMessageService = null; + this.scheduleMessageService = null; + } + + // load过程依赖此服务，所以提前启动 + this.allocateMapedFileService.start(); + this.dispatchMessageService.start(); + // 因为下面的recover会分发请求到索引服务，如果不启动，分发过程会被流控 + this.indexService.start(); + } + + + public void truncateDirtyLogicFiles(long phyOffset) { + ConcurrentHashMap> tables = + DefaultMessageStore.this.consumeQueueTable; + + for (ConcurrentHashMap maps : tables.values()) { + for (ConsumeQueue logic : maps.values()) { + logic.truncateDirtyLogicFiles(phyOffset); + } + } + } + + + /** + * 加载数据 + * + * @throws IOException + */ + public boolean load() { + boolean result = true; + + try { + boolean lastExitOK = !this.isTempFileExist(); + log.info("last shutdown {}", (lastExitOK ? "normally" : "abnormally")); + + // load 定时进度 + // 这个步骤要放置到最前面，从CommitLog里Recover定时消息需要依赖加载的定时级别参数 + // slave依赖scheduleMessageService做定时消息的恢复 + if (null != scheduleMessageService) { + result = result && this.scheduleMessageService.load(); + } + + // load Commit Log + result = result && this.commitLog.load(); + + // load Consume Queue + result = result && this.loadConsumeQueue(); + + if (result) { + this.storeCheckpoint = + new StoreCheckpoint(StorePathConfigHelper.getStoreCheckpoint(this.messageStoreConfig + .getStorePathRootDir())); + + this.indexService.load(lastExitOK); + + // 尝试恢复数据 + this.recover(lastExitOK); + + log.info("load over, and the max phy offset = {}", this.getMaxPhyOffset()); + } + } + catch (Exception e) { + log.error("load exception", e); + result = false; + } + + if (!result) { + this.allocateMapedFileService.shutdown(); + } + + return result; + } + + + private void addScheduleTask() { + // 定时删除过期文件 + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + DefaultMessageStore.this.cleanFilesPeriodically(); + } + }, 1000 * 60, this.messageStoreConfig.getCleanResourceInterval(), TimeUnit.MILLISECONDS); + + // 定时清理完全不使用的队列 + // this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + // @Override + // public void run() { + // DefaultMessageStore.this.cleanExpiredConsumerQueue(); + // } + // }, 1, 1, TimeUnit.HOURS); + } + + + private void cleanFilesPeriodically() { + this.cleanCommitLogService.run(); + this.cleanConsumeQueueService.run(); + } + + + public void cleanExpiredConsumerQueue() { + // CommitLog的最小Offset + long minCommitLogOffset = this.commitLog.getMinOffset(); + + Iterator>> it = + this.consumeQueueTable.entrySet().iterator(); + while (it.hasNext()) { + Entry> next = it.next(); + String topic = next.getKey(); + if (!topic.equals(ScheduleMessageService.SCHEDULE_TOPIC)) { + ConcurrentHashMap queueTable = next.getValue(); + Iterator> itQT = queueTable.entrySet().iterator(); + while (itQT.hasNext()) { + Entry nextQT = itQT.next(); + long maxCLOffsetInConsumeQueue = nextQT.getValue().getLastOffset(); + + // maxCLOffsetInConsumeQueue==-1有可能正好是索引文件刚好创建的那一时刻,此时不清除数据 + if (maxCLOffsetInConsumeQueue == -1) { + log.warn( + "maybe ConsumeQueue was created just now. topic={} queueId={} maxPhysicOffset={} minLogicOffset={}.",// + nextQT.getValue().getTopic(),// + nextQT.getValue().getQueueId(),// + nextQT.getValue().getMaxPhysicOffset(),// + nextQT.getValue().getMinLogicOffset()); + } + else if (maxCLOffsetInConsumeQueue < minCommitLogOffset) { + log.info( + "cleanExpiredConsumerQueue: {} {} consumer queue destroyed, minCommitLogOffset: {} maxCLOffsetInConsumeQueue: {}",// + topic,// + nextQT.getKey(),// + minCommitLogOffset,// + maxCLOffsetInConsumeQueue); + + DefaultMessageStore.this.commitLog.removeQueurFromTopicQueueTable(nextQT.getValue() + .getTopic(), nextQT.getValue().getQueueId()); + + nextQT.getValue().destroy(); + itQT.remove(); + } + } + + if (queueTable.isEmpty()) { + log.info("cleanExpiredConsumerQueue: {},topic destroyed", topic); + it.remove(); + } + } + } + } + + + /** + * 启动存储服务 + * + * @throws Exception + */ + public void start() throws Exception { + // 在构造函数已经start了。 + // this.indexService.start(); + // 在构造函数已经start了。 + // this.dispatchMessageService.start(); + this.flushConsumeQueueService.start(); + this.commitLog.start(); + this.storeStatsService.start(); + + // slave不启动scheduleMessageService避免对消费队列的并发操作 + if (this.scheduleMessageService != null && SLAVE != messageStoreConfig.getBrokerRole()) { + this.scheduleMessageService.start(); + } + + if (this.reputMessageService != null) { + this.reputMessageService.setReputFromOffset(this.commitLog.getMaxOffset()); + this.reputMessageService.start(); + } + + this.haService.start(); + + this.createTempFile(); + this.addScheduleTask(); + this.shutdown = false; + } + + + /** + * 关闭存储服务 + */ + public void shutdown() { + if (!this.shutdown) { + this.shutdown = true; + + this.scheduledExecutorService.shutdown(); + + try { + // 等待其他调用停止 + Thread.sleep(1000 * 3); + } + catch (InterruptedException e) { + log.error("shutdown Exception, ", e); + } + + if (this.scheduleMessageService != null) { + this.scheduleMessageService.shutdown(); + } + + this.haService.shutdown(); + + this.storeStatsService.shutdown(); + this.dispatchMessageService.shutdown(); + this.indexService.shutdown(); + this.flushConsumeQueueService.shutdown(); + this.commitLog.shutdown(); + this.allocateMapedFileService.shutdown(); + if (this.reputMessageService != null) { + this.reputMessageService.shutdown(); + } + this.storeCheckpoint.flush(); + this.storeCheckpoint.shutdown(); + + this.deleteFile(StorePathConfigHelper.getAbortFile(this.messageStoreConfig.getStorePathRootDir())); + } + } + + + public void destroy() { + this.destroyLogics(); + this.commitLog.destroy(); + this.indexService.destroy(); + this.deleteFile(StorePathConfigHelper.getAbortFile(this.messageStoreConfig.getStorePathRootDir())); + this.deleteFile(StorePathConfigHelper.getStoreCheckpoint(this.messageStoreConfig + .getStorePathRootDir())); + } + + + public void destroyLogics() { + for (ConcurrentHashMap maps : this.consumeQueueTable.values()) { + for (ConsumeQueue logic : maps.values()) { + logic.destroy(); + } + } + } + + + public PutMessageResult putMessage(MessageExtBrokerInner msg) { + if (this.shutdown) { + log.warn("message store has shutdown, so putMessage is forbidden"); + return new PutMessageResult(PutMessageStatus.SERVICE_NOT_AVAILABLE, null); + } + + if (BrokerRole.SLAVE == this.messageStoreConfig.getBrokerRole()) { + long value = this.printTimes.getAndIncrement(); + if ((value % 50000) == 0) { + log.warn("message store is slave mode, so putMessage is forbidden "); + } + + return new PutMessageResult(PutMessageStatus.SERVICE_NOT_AVAILABLE, null); + } + + if (!this.runningFlags.isWriteable()) { + long value = this.printTimes.getAndIncrement(); + if ((value % 50000) == 0) { + log.warn("message store is not writeable, so putMessage is forbidden " + + this.runningFlags.getFlagBits()); + } + + return new PutMessageResult(PutMessageStatus.SERVICE_NOT_AVAILABLE, null); + } + else { + this.printTimes.set(0); + } + + // message topic长度校验 + if (msg.getTopic().length() > Byte.MAX_VALUE) { + log.warn("putMessage message topic length too long " + msg.getTopic().length()); + return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, null); + } + + // message properties长度校验 + if (msg.getPropertiesString() != null && msg.getPropertiesString().length() > Short.MAX_VALUE) { + log.warn("putMessage message properties length too long " + msg.getPropertiesString().length()); + return new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, null); + } + + long beginTime = this.getSystemClock().now(); + PutMessageResult result = this.commitLog.putMessage(msg); + // 性能数据统计 + long eclipseTime = this.getSystemClock().now() - beginTime; + if (eclipseTime > 1000) { + log.warn("putMessage not in lock eclipse time(ms) " + eclipseTime); + } + this.storeStatsService.setPutMessageEntireTimeMax(eclipseTime); + this.storeStatsService.getSinglePutMessageTopicTimesTotal(msg.getTopic()).incrementAndGet(); + + if (null == result || !result.isOk()) { + this.storeStatsService.getPutMessageFailedTimes().incrementAndGet(); + } + + return result; + } + + + public SystemClock getSystemClock() { + return systemClock; + } + + + public GetMessageResult getMessage(final String group, final String topic, final int queueId, + final long offset, final int maxMsgNums, final SubscriptionData subscriptionData) { + if (this.shutdown) { + log.warn("message store has shutdown, so getMessage is forbidden"); + return null; + } + + if (!this.runningFlags.isReadable()) { + log.warn("message store is not readable, so getMessage is forbidden " + + this.runningFlags.getFlagBits()); + return null; + } + + long beginTime = this.getSystemClock().now(); + + // 枚举变量，取消息结果 + GetMessageStatus status = GetMessageStatus.NO_MESSAGE_IN_QUEUE; + // 当被过滤后，返回下一次开始的Offset + long nextBeginOffset = offset; + // 逻辑队列中的最小Offset + long minOffset = 0; + // 逻辑队列中的最大Offset + long maxOffset = 0; + + GetMessageResult getResult = new GetMessageResult(); + + ConsumeQueue consumeQueue = findConsumeQueue(topic, queueId); + if (consumeQueue != null) { + minOffset = consumeQueue.getMinOffsetInQuque(); + maxOffset = consumeQueue.getMaxOffsetInQuque(); + + if (maxOffset == 0) { + status = GetMessageStatus.NO_MESSAGE_IN_QUEUE; + nextBeginOffset = 0; + } + else if (offset < minOffset) { + status = GetMessageStatus.OFFSET_TOO_SMALL; + nextBeginOffset = minOffset; + } + else if (offset == maxOffset) { + status = GetMessageStatus.OFFSET_OVERFLOW_ONE; + nextBeginOffset = offset; + } + else if (offset > maxOffset) { + status = GetMessageStatus.OFFSET_OVERFLOW_BADLY; + if (0 == minOffset) { + nextBeginOffset = minOffset; + } + else { + nextBeginOffset = maxOffset; + } + } + else { + SelectMapedBufferResult bufferConsumeQueue = consumeQueue.getIndexBuffer(offset); + if (bufferConsumeQueue != null) { + try { + status = GetMessageStatus.NO_MATCHED_MESSAGE; + + long nextPhyFileStartOffset = Long.MIN_VALUE; + long maxPhyOffsetPulling = 0; + + int i = 0; + final int MaxFilterMessageCount = 16000; + for (; i < bufferConsumeQueue.getSize() && i < MaxFilterMessageCount; i += + ConsumeQueue.CQStoreUnitSize) { + long offsetPy = bufferConsumeQueue.getByteBuffer().getLong(); + int sizePy = bufferConsumeQueue.getByteBuffer().getInt(); + long tagsCode = bufferConsumeQueue.getByteBuffer().getLong(); + + maxPhyOffsetPulling = offsetPy; + + // 说明物理文件正在被删除 + if (nextPhyFileStartOffset != Long.MIN_VALUE) { + if (offsetPy < nextPhyFileStartOffset) + continue; + } + + // 判断是否拉磁盘数据 + boolean isInDisk = checkInDiskByCommitOffset(offsetPy); + // 此批消息达到上限了 + if (this.isTheBatchFull(sizePy, maxMsgNums, getResult.getBufferTotalSize(), + getResult.getMessageCount(), isInDisk)) { + break; + } + + // 消息过滤 + if (this.messageFilter.isMessageMatched(subscriptionData, tagsCode)) { + SelectMapedBufferResult selectResult = + this.commitLog.getMessage(offsetPy, sizePy); + if (selectResult != null) { + this.storeStatsService.getGetMessageTransferedMsgCount() + .incrementAndGet(); + getResult.addMessage(selectResult); + status = GetMessageStatus.FOUND; + nextPhyFileStartOffset = Long.MIN_VALUE; + + // 统计消息数据 + if (isInDisk && brokerStatsManager != null) { + brokerStatsManager.incGroupGetFromDiskNums(group, topic, 1); + brokerStatsManager.incGroupGetFromDiskSize(group, topic, + selectResult.getSize()); + brokerStatsManager.incBrokerGetFromDiskNums(1); + } + } + else { + if (getResult.getBufferTotalSize() == 0) { + status = GetMessageStatus.MESSAGE_WAS_REMOVING; + } + + // 物理文件正在被删除，尝试跳过 + nextPhyFileStartOffset = this.commitLog.rollNextFile(offsetPy); + } + } + else { + if (getResult.getBufferTotalSize() == 0) { + status = GetMessageStatus.NO_MATCHED_MESSAGE; + } + + if (log.isDebugEnabled()) { + log.debug("message type not matched, client: " + subscriptionData + + " server: " + tagsCode); + } + } + } + + nextBeginOffset = offset + (i / ConsumeQueue.CQStoreUnitSize); + + // TODO 是否会影响性能，需要测试 + long diff = this.getMaxPhyOffset() - maxPhyOffsetPulling; + long memory = + (long) (StoreUtil.TotalPhysicalMemorySize * (this.messageStoreConfig + .getAccessMessageInMemoryMaxRatio() / 100.0)); + getResult.setSuggestPullingFromSlave(diff > memory); + } + finally { + // 必须释放资源 + bufferConsumeQueue.release(); + } + } + else { + status = GetMessageStatus.OFFSET_FOUND_NULL; + nextBeginOffset = consumeQueue.rollNextFile(offset); + log.warn("consumer request topic: " + topic + "offset: " + offset + " minOffset: " + + minOffset + " maxOffset: " + maxOffset + ", but access logic queue failed."); + } + } + } + // 请求的队列Id没有 + else { + status = GetMessageStatus.NO_MATCHED_LOGIC_QUEUE; + nextBeginOffset = 0; + } + + if (GetMessageStatus.FOUND == status) { + this.storeStatsService.getGetMessageTimesTotalFound().incrementAndGet(); + } + else { + this.storeStatsService.getGetMessageTimesTotalMiss().incrementAndGet(); + } + long eclipseTime = this.getSystemClock().now() - beginTime; + this.storeStatsService.setGetMessageEntireTimeMax(eclipseTime); + + getResult.setStatus(status); + getResult.setNextBeginOffset(nextBeginOffset); + getResult.setMaxOffset(maxOffset); + getResult.setMinOffset(minOffset); + return getResult; + } + + + /** + * 返回的是当前队列的最大Offset，这个Offset没有对应的消息 + */ + public long getMaxOffsetInQuque(String topic, int queueId) { + ConsumeQueue logic = this.findConsumeQueue(topic, queueId); + if (logic != null) { + long offset = logic.getMaxOffsetInQuque(); + return offset; + } + + return 0; + } + + + /** + * 返回的是当前队列的最小Offset + */ + public long getMinOffsetInQuque(String topic, int queueId) { + ConsumeQueue logic = this.findConsumeQueue(topic, queueId); + if (logic != null) { + return logic.getMinOffsetInQuque(); + } + + return -1; + } + + + public long getOffsetInQueueByTime(String topic, int queueId, long timestamp) { + ConsumeQueue logic = this.findConsumeQueue(topic, queueId); + if (logic != null) { + return logic.getOffsetInQueueByTime(timestamp); + } + + return 0; + } + + + public MessageExt lookMessageByOffset(long commitLogOffset) { + SelectMapedBufferResult sbr = this.commitLog.getMessage(commitLogOffset, 4); + if (null != sbr) { + try { + // 1 TOTALSIZE + int size = sbr.getByteBuffer().getInt(); + return lookMessageByOffset(commitLogOffset, size); + } + finally { + sbr.release(); + } + } + + return null; + } + + + @Override + public SelectMapedBufferResult selectOneMessageByOffset(long commitLogOffset) { + SelectMapedBufferResult sbr = this.commitLog.getMessage(commitLogOffset, 4); + if (null != sbr) { + try { + // 1 TOTALSIZE + int size = sbr.getByteBuffer().getInt(); + return this.commitLog.getMessage(commitLogOffset, size); + } + finally { + sbr.release(); + } + } + + return null; + } + + + @Override + public SelectMapedBufferResult selectOneMessageByOffset(long commitLogOffset, int msgSize) { + return this.commitLog.getMessage(commitLogOffset, msgSize); + } + + + public String getRunningDataInfo() { + return this.storeStatsService.toString(); + } + + + @Override + public HashMap getRuntimeInfo() { + HashMap result = this.storeStatsService.getRuntimeInfo(); + // 检测物理文件磁盘空间 + { + String storePathPhysic = DefaultMessageStore.this.getMessageStoreConfig().getStorePathCommitLog(); + double physicRatio = UtilAll.getDiskPartitionSpaceUsedPercent(storePathPhysic); + result.put(RunningStats.commitLogDiskRatio.name(), String.valueOf(physicRatio)); + + } + + // 检测逻辑文件磁盘空间 + { + + String storePathLogics = + StorePathConfigHelper.getStorePathConsumeQueue(this.messageStoreConfig + .getStorePathRootDir()); + double logicsRatio = UtilAll.getDiskPartitionSpaceUsedPercent(storePathLogics); + result.put(RunningStats.consumeQueueDiskRatio.name(), String.valueOf(logicsRatio)); + } + + // 延时进度 + { + if (this.scheduleMessageService != null) { + this.scheduleMessageService.buildRunningStats(result); + } + } + + result.put(RunningStats.commitLogMinOffset.name(), + String.valueOf(DefaultMessageStore.this.getMinPhyOffset())); + result.put(RunningStats.commitLogMaxOffset.name(), + String.valueOf(DefaultMessageStore.this.getMaxPhyOffset())); + + return result; + } + + + @Override + public long getMaxPhyOffset() { + return this.commitLog.getMaxOffset(); + } + + + @Override + public long getEarliestMessageTime(String topic, int queueId) { + ConsumeQueue logicQueue = this.findConsumeQueue(topic, queueId); + if (logicQueue != null) { + long minLogicOffset = logicQueue.getMinLogicOffset(); + + SelectMapedBufferResult result = + logicQueue.getIndexBuffer(minLogicOffset / ConsumeQueue.CQStoreUnitSize); + if (result != null) { + try { + final long phyOffset = result.getByteBuffer().getLong(); + final int size = result.getByteBuffer().getInt(); + long storeTime = this.getCommitLog().pickupStoretimestamp(phyOffset, size); + return storeTime; + } + catch (Exception e) { + } + finally { + result.release(); + } + } + } + + return -1; + } + + + @Override + public long getMessageStoreTimeStamp(String topic, int queueId, long offset) { + ConsumeQueue logicQueue = this.findConsumeQueue(topic, queueId); + if (logicQueue != null) { + SelectMapedBufferResult result = logicQueue.getIndexBuffer(offset); + if (result != null) { + try { + final long phyOffset = result.getByteBuffer().getLong(); + final int size = result.getByteBuffer().getInt(); + long storeTime = this.getCommitLog().pickupStoretimestamp(phyOffset, size); + return storeTime; + } + catch (Exception e) { + } + finally { + result.release(); + } + } + } + + return -1; + } + + + @Override + public long getMessageTotalInQueue(String topic, int queueId) { + ConsumeQueue logicQueue = this.findConsumeQueue(topic, queueId); + if (logicQueue != null) { + return logicQueue.getMessageTotalInQueue(); + } + + return -1; + } + + + @Override + public SelectMapedBufferResult getCommitLogData(final long offset) { + if (this.shutdown) { + log.warn("message store has shutdown, so getPhyQueueData is forbidden"); + return null; + } + + return this.commitLog.getData(offset); + } + + + @Override + public boolean appendToCommitLog(long startOffset, byte[] data) { + if (this.shutdown) { + log.warn("message store has shutdown, so appendToPhyQueue is forbidden"); + return false; + } + + boolean result = this.commitLog.appendData(startOffset, data); + if (result) { + this.reputMessageService.wakeup(); + } + else { + log.error("appendToPhyQueue failed " + startOffset + " " + data.length); + } + + return result; + } + + + @Override + public void excuteDeleteFilesManualy() { + this.cleanCommitLogService.excuteDeleteFilesManualy(); + } + + + @Override + public QueryMessageResult queryMessage(String topic, String key, int maxNum, long begin, long end) { + QueryMessageResult queryMessageResult = new QueryMessageResult(); + + long lastQueryMsgTime = end; + + for (int i = 0; i < 3; i++) { + QueryOffsetResult queryOffsetResult = + this.indexService.queryOffset(topic, key, maxNum, begin, lastQueryMsgTime); + if (queryOffsetResult.getPhyOffsets().isEmpty()) { + break; + } + + // 从小到达排序 + Collections.sort(queryOffsetResult.getPhyOffsets()); + + queryMessageResult.setIndexLastUpdatePhyoffset(queryOffsetResult.getIndexLastUpdatePhyoffset()); + queryMessageResult.setIndexLastUpdateTimestamp(queryOffsetResult.getIndexLastUpdateTimestamp()); + + for (int m = 0; m < queryOffsetResult.getPhyOffsets().size(); m++) { + long offset = queryOffsetResult.getPhyOffsets().get(m); + + try { + // 在服务器检验Hash冲突 + boolean match = true; + MessageExt msg = this.lookMessageByOffset(offset); + if (0 == m) { + lastQueryMsgTime = msg.getStoreTimestamp(); + } + + String[] keyArray = msg.getKeys().split(MessageConst.KEY_SEPARATOR); + if (topic.equals(msg.getTopic())) { + for (String k : keyArray) { + if (k.equals(key)) { + match = true; + break; + } + } + } + + if (match) { + SelectMapedBufferResult result = this.commitLog.getData(offset, false); + if (result != null) { + int size = result.getByteBuffer().getInt(0); + result.getByteBuffer().limit(size); + result.setSize(size); + queryMessageResult.addMessage(result); + } + } + else { + log.warn("queryMessage hash duplicate, {} {}", topic, key); + } + } + catch (Exception e) { + log.error("queryMessage exception", e); + } + } + + // 只要查到记录就返回 + if (queryMessageResult.getBufferTotalSize() > 0) { + break; + } + + // 都遍历完了， 但是没有找到消息 + if (lastQueryMsgTime < begin) { + break; + } + } + + return queryMessageResult; + } + + + @Override + public void updateHaMasterAddress(String newAddr) { + this.haService.updateMasterAddress(newAddr); + } + + + @Override + public long now() { + return this.systemClock.now(); + } + + + public CommitLog getCommitLog() { + return commitLog; + } + + + public MessageExt lookMessageByOffset(long commitLogOffset, int size) { + SelectMapedBufferResult sbr = this.commitLog.getMessage(commitLogOffset, size); + if (null != sbr) { + try { + return MessageDecoder.decode(sbr.getByteBuffer(), true, false); + } + finally { + sbr.release(); + } + } + + return null; + } + + + public ConsumeQueue findConsumeQueue(String topic, int queueId) { + ConcurrentHashMap map = consumeQueueTable.get(topic); + if (null == map) { + ConcurrentHashMap newMap = + new ConcurrentHashMap(128); + ConcurrentHashMap oldMap = consumeQueueTable.putIfAbsent(topic, newMap); + if (oldMap != null) { + map = oldMap; + } + else { + map = newMap; + } + } + + ConsumeQueue logic = map.get(queueId); + if (null == logic) { + ConsumeQueue newLogic = + new ConsumeQueue(// + topic,// + queueId,// + StorePathConfigHelper.getStorePathConsumeQueue(this.messageStoreConfig + .getStorePathRootDir()),// + this.getMessageStoreConfig().getMapedFileSizeConsumeQueue(),// + this); + ConsumeQueue oldLogic = map.putIfAbsent(queueId, newLogic); + if (oldLogic != null) { + logic = oldLogic; + } + else { + logic = newLogic; + } + } + + return logic; + } + + + private boolean isTheBatchFull(int sizePy, int maxMsgNums, int bufferTotal, int messageTotal, + boolean isInDisk) { + // 第一条消息可以不做限制 + if (0 == bufferTotal || 0 == messageTotal) { + return false; + } + + if ((messageTotal + 1) >= maxMsgNums) { + return true; + } + + // 消息在磁盘 + if (isInDisk) { + if ((bufferTotal + sizePy) > this.messageStoreConfig.getMaxTransferBytesOnMessageInDisk()) { + return true; + } + + if ((messageTotal + 1) > this.messageStoreConfig.getMaxTransferCountOnMessageInDisk()) { + return true; + } + } + // 消息在内存 + else { + if ((bufferTotal + sizePy) > this.messageStoreConfig.getMaxTransferBytesOnMessageInMemory()) { + return true; + } + + if ((messageTotal + 1) > this.messageStoreConfig.getMaxTransferCountOnMessageInMemory()) { + return true; + } + } + + return false; + } + + + private void deleteFile(final String fileName) { + File file = new File(fileName); + boolean result = file.delete(); + log.info(fileName + (result ? " delete OK" : " delete Failed")); + } + + + /** + * 启动服务后，在存储根目录创建临时文件，类似于 UNIX VI编辑工具 + * + * @throws IOException + */ + private void createTempFile() throws IOException { + String fileName = StorePathConfigHelper.getAbortFile(this.messageStoreConfig.getStorePathRootDir()); + File file = new File(fileName); + MapedFile.ensureDirOK(file.getParent()); + boolean result = file.createNewFile(); + log.info(fileName + (result ? " create OK" : " already exists")); + } + + + private boolean isTempFileExist() { + String fileName = StorePathConfigHelper.getAbortFile(this.messageStoreConfig.getStorePathRootDir()); + File file = new File(fileName); + return file.exists(); + } + + + private boolean loadConsumeQueue() { + File dirLogic = + new File(StorePathConfigHelper.getStorePathConsumeQueue(this.messageStoreConfig + .getStorePathRootDir())); + File[] fileTopicList = dirLogic.listFiles(); + if (fileTopicList != null) { + // TOPIC 遍历 + for (File fileTopic : fileTopicList) { + String topic = fileTopic.getName(); + // TOPIC 下队列遍历 + File[] fileQueueIdList = fileTopic.listFiles(); + if (fileQueueIdList != null) { + for (File fileQueueId : fileQueueIdList) { + int queueId = Integer.parseInt(fileQueueId.getName()); + ConsumeQueue logic = + new ConsumeQueue(// + topic,// + queueId,// + StorePathConfigHelper.getStorePathConsumeQueue(this.messageStoreConfig + .getStorePathRootDir()),// + this.getMessageStoreConfig().getMapedFileSizeConsumeQueue(),// + this); + this.putConsumeQueue(topic, queueId, logic); + if (!logic.load()) { + return false; + } + } + } + } + } + + log.info("load logics queue all over, OK"); + + return true; + } + + + public MessageStoreConfig getMessageStoreConfig() { + return messageStoreConfig; + } + + + private void putConsumeQueue(final String topic, final int queueId, final ConsumeQueue consumeQueue) { + ConcurrentHashMap map = this.consumeQueueTable.get(topic); + if (null == map) { + map = new ConcurrentHashMap(); + map.put(queueId, consumeQueue); + this.consumeQueueTable.put(topic, map); + } + else { + map.put(queueId, consumeQueue); + } + } + + + private void recover(final boolean lastExitOK) { + // 先按照正常流程恢复Consume Queue + this.recoverConsumeQueue(); + + // 正常数据恢复 + if (lastExitOK) { + this.commitLog.recoverNormally(); + } + // 异常数据恢复，OS CRASH或者JVM CRASH或者机器掉电 + else { + this.commitLog.recoverAbnormally(); + } + + // 保证消息都能从DispatchService缓冲队列进入到真正的队列 + while (this.dispatchMessageService.hasRemainMessage()) { + try { + Thread.sleep(500); + log.info("waiting dispatching message over"); + } + catch (InterruptedException e) { + } + } + + this.recoverTopicQueueTable(); + } + + + private void recoverTopicQueueTable() { + HashMap table = new HashMap(1024); + long minPhyOffset = this.commitLog.getMinOffset(); + for (ConcurrentHashMap maps : this.consumeQueueTable.values()) { + for (ConsumeQueue logic : maps.values()) { + // 恢复写入消息时，记录的队列offset + String key = logic.getTopic() + "-" + logic.getQueueId(); + table.put(key, logic.getMaxOffsetInQuque()); + // 恢复每个队列的最小offset + logic.correctMinOffset(minPhyOffset); + } + } + + this.commitLog.setTopicQueueTable(table); + } + + + private void recoverConsumeQueue() { + for (ConcurrentHashMap maps : this.consumeQueueTable.values()) { + for (ConsumeQueue logic : maps.values()) { + logic.recover(); + } + } + } + + + public void putMessagePostionInfo(String topic, int queueId, long offset, int size, long tagsCode, + long storeTimestamp, long logicOffset) { + ConsumeQueue cq = this.findConsumeQueue(topic, queueId); + cq.putMessagePostionInfoWrapper(offset, size, tagsCode, storeTimestamp, logicOffset); + } + + + public void putDispatchRequest(final DispatchRequest dispatchRequest) { + this.dispatchMessageService.putRequest(dispatchRequest); + } + + + public DispatchMessageService getDispatchMessageService() { + return dispatchMessageService; + } + + + public AllocateMapedFileService getAllocateMapedFileService() { + return allocateMapedFileService; + } + + + public StoreStatsService getStoreStatsService() { + return storeStatsService; + } + + + public RunningFlags getAccessRights() { + return runningFlags; + } + + + public ConcurrentHashMap> getConsumeQueueTable() { + return consumeQueueTable; + } + + + public StoreCheckpoint getStoreCheckpoint() { + return storeCheckpoint; + } + + + public HAService getHaService() { + return haService; + } + + + public ScheduleMessageService getScheduleMessageService() { + return scheduleMessageService; + } + + + public RunningFlags getRunningFlags() { + return runningFlags; + } + + /** + * 清理物理文件服务 + */ + class CleanCommitLogService { + // 手工触发一次最多删除次数 + private final static int MaxManualDeleteFileTimes = 20; + // 磁盘空间警戒水位，超过，则停止接收新消息（出于保护自身目的） + private final double DiskSpaceWarningLevelRatio = Double.parseDouble(System.getProperty( + "rocketmq.broker.diskSpaceWarningLevelRatio", "0.90")); + // 磁盘空间强制删除文件水位 + private final double DiskSpaceCleanForciblyRatio = Double.parseDouble(System.getProperty( + "rocketmq.broker.diskSpaceCleanForciblyRatio", "0.85")); + private long lastRedeleteTimestamp = 0; + // 手工触发删除消息 + private volatile int manualDeleteFileSeveralTimes = 0; + // 立刻开始强制删除文件 + private volatile boolean cleanImmediately = false; + + + public void excuteDeleteFilesManualy() { + this.manualDeleteFileSeveralTimes = MaxManualDeleteFileTimes; + DefaultMessageStore.log.info("excuteDeleteFilesManualy was invoked"); + } + + + public void run() { + try { + this.deleteExpiredFiles(); + + this.redeleteHangedFile(); + } + catch (Exception e) { + DefaultMessageStore.log.warn(this.getServiceName() + " service has exception. ", e); + } + } + + + public String getServiceName() { + return CleanCommitLogService.class.getSimpleName(); + } + + + /** + * 最前面的文件有可能Hang住，定期检查一下 + */ + private void redeleteHangedFile() { + int interval = DefaultMessageStore.this.getMessageStoreConfig().getRedeleteHangedFileInterval(); + long currentTimestamp = System.currentTimeMillis(); + if ((currentTimestamp - this.lastRedeleteTimestamp) > interval) { + this.lastRedeleteTimestamp = currentTimestamp; + int destroyMapedFileIntervalForcibly = + DefaultMessageStore.this.getMessageStoreConfig() + .getDestroyMapedFileIntervalForcibly(); + if (DefaultMessageStore.this.commitLog.retryDeleteFirstFile(destroyMapedFileIntervalForcibly)) { + // TODO + } + } + } + + + private void deleteExpiredFiles() { + int deleteCount = 0; + long fileReservedTime = DefaultMessageStore.this.getMessageStoreConfig().getFileReservedTime(); + int deletePhysicFilesInterval = + DefaultMessageStore.this.getMessageStoreConfig().getDeleteCommitLogFilesInterval(); + int destroyMapedFileIntervalForcibly = + DefaultMessageStore.this.getMessageStoreConfig().getDestroyMapedFileIntervalForcibly(); + + boolean timeup = this.isTimeToDelete(); + boolean spacefull = this.isSpaceToDelete(); + boolean manualDelete = this.manualDeleteFileSeveralTimes > 0; + + // 删除物理队列文件 + if (timeup || spacefull || manualDelete) { + + if (manualDelete) + this.manualDeleteFileSeveralTimes--; + + // 是否立刻强制删除文件 + boolean cleanAtOnce = + DefaultMessageStore.this.getMessageStoreConfig().isCleanFileForciblyEnable() + && this.cleanImmediately; + + log.info( + "begin to delete before {} hours file. timeup: {} spacefull: {} manualDeleteFileSeveralTimes: {} cleanAtOnce: {}",// + fileReservedTime,// + timeup,// + spacefull,// + manualDeleteFileSeveralTimes,// + cleanAtOnce); + + // 小时转化成毫秒 + fileReservedTime *= 60 * 60 * 1000; + + deleteCount = + DefaultMessageStore.this.commitLog.deleteExpiredFile(fileReservedTime, + deletePhysicFilesInterval, destroyMapedFileIntervalForcibly, cleanAtOnce); + if (deleteCount > 0) { + // TODO + } + // 危险情况：磁盘满了，但是又无法删除文件 + else if (spacefull) { + // XXX: warn and notify me + log.warn("disk space will be full soon, but delete file failed."); + } + } + } + + + /** + * 是否可以删除文件，空间是否满足 + */ + private boolean isSpaceToDelete() { + double ratio = + DefaultMessageStore.this.getMessageStoreConfig().getDiskMaxUsedSpaceRatio() / 100.0; + + cleanImmediately = false; + + // 检测物理文件磁盘空间 + { + String storePathPhysic = + DefaultMessageStore.this.getMessageStoreConfig().getStorePathCommitLog(); + double physicRatio = UtilAll.getDiskPartitionSpaceUsedPercent(storePathPhysic); + if (physicRatio > DiskSpaceWarningLevelRatio) { + boolean diskok = DefaultMessageStore.this.runningFlags.getAndMakeDiskFull(); + if (diskok) { + DefaultMessageStore.log.error("physic disk maybe full soon " + physicRatio + + ", so mark disk full"); + System.gc(); + } + + cleanImmediately = true; + } + else if (physicRatio > DiskSpaceCleanForciblyRatio) { + cleanImmediately = true; + } + else { + boolean diskok = DefaultMessageStore.this.runningFlags.getAndMakeDiskOK(); + if (!diskok) { + DefaultMessageStore.log.info("physic disk space OK " + physicRatio + + ", so mark disk ok"); + } + } + + if (physicRatio < 0 || physicRatio > ratio) { + DefaultMessageStore.log.info("physic disk maybe full soon, so reclaim space, " + + physicRatio); + return true; + } + } + + // 检测逻辑文件磁盘空间 + { + String storePathLogics = + StorePathConfigHelper.getStorePathConsumeQueue(DefaultMessageStore.this + .getMessageStoreConfig().getStorePathRootDir()); + double logicsRatio = UtilAll.getDiskPartitionSpaceUsedPercent(storePathLogics); + if (logicsRatio > DiskSpaceWarningLevelRatio) { + boolean diskok = DefaultMessageStore.this.runningFlags.getAndMakeDiskFull(); + if (diskok) { + DefaultMessageStore.log.error("logics disk maybe full soon " + logicsRatio + + ", so mark disk full"); + System.gc(); + } + + cleanImmediately = true; + } + else if (logicsRatio > DiskSpaceCleanForciblyRatio) { + cleanImmediately = true; + } + else { + boolean diskok = DefaultMessageStore.this.runningFlags.getAndMakeDiskOK(); + if (!diskok) { + DefaultMessageStore.log.info("logics disk space OK " + logicsRatio + + ", so mark disk ok"); + } + } + + if (logicsRatio < 0 || logicsRatio > ratio) { + DefaultMessageStore.log.info("logics disk maybe full soon, so reclaim space, " + + logicsRatio); + return true; + } + } + + return false; + } + + + /** + * 是否可以删除文件，时间是否满足 + */ + private boolean isTimeToDelete() { + String when = DefaultMessageStore.this.getMessageStoreConfig().getDeleteWhen(); + if (UtilAll.isItTimeToDo(when)) { + DefaultMessageStore.log.info("it's time to reclaim disk space, " + when); + return true; + } + + return false; + } + + + public int getManualDeleteFileSeveralTimes() { + return manualDeleteFileSeveralTimes; + } + + + public void setManualDeleteFileSeveralTimes(int manualDeleteFileSeveralTimes) { + this.manualDeleteFileSeveralTimes = manualDeleteFileSeveralTimes; + } + } + + /** + * 清理逻辑文件服务 + */ + class CleanConsumeQueueService { + private long lastPhysicalMinOffset = 0; + + + private void deleteExpiredFiles() { + int deleteLogicsFilesInterval = + DefaultMessageStore.this.getMessageStoreConfig().getDeleteConsumeQueueFilesInterval(); + + long minOffset = DefaultMessageStore.this.commitLog.getMinOffset(); + if (minOffset > this.lastPhysicalMinOffset) { + this.lastPhysicalMinOffset = minOffset; + + // 删除逻辑队列文件 + ConcurrentHashMap> tables = + DefaultMessageStore.this.consumeQueueTable; + + for (ConcurrentHashMap maps : tables.values()) { + for (ConsumeQueue logic : maps.values()) { + int deleteCount = logic.deleteExpiredFile(minOffset); + + if (deleteCount > 0 && deleteLogicsFilesInterval > 0) { + try { + Thread.sleep(deleteLogicsFilesInterval); + } + catch (InterruptedException e) { + } + } + } + } + + // 删除索引 + DefaultMessageStore.this.indexService.deleteExpiredFile(minOffset); + } + } + + + public void run() { + try { + this.deleteExpiredFiles(); + } + catch (Exception e) { + DefaultMessageStore.log.warn(this.getServiceName() + " service has exception. ", e); + } + } + + + public String getServiceName() { + return CleanConsumeQueueService.class.getSimpleName(); + } + } + + /** + * 逻辑队列刷盘服务 + */ + class FlushConsumeQueueService extends ServiceThread { + private static final int RetryTimesOver = 3; + private long lastFlushTimestamp = 0; + + + private void doFlush(int retryTimes) { + /** + * 变量含义：如果大于0，则标识这次刷盘必须刷多少个page，如果=0，则有多少刷多少 + */ + int flushConsumeQueueLeastPages = + DefaultMessageStore.this.getMessageStoreConfig().getFlushConsumeQueueLeastPages(); + + if (retryTimes == RetryTimesOver) { + flushConsumeQueueLeastPages = 0; + } + + long logicsMsgTimestamp = 0; + + // 定时刷盘 + int flushConsumeQueueThoroughInterval = + DefaultMessageStore.this.getMessageStoreConfig().getFlushConsumeQueueThoroughInterval(); + long currentTimeMillis = System.currentTimeMillis(); + if (currentTimeMillis >= (this.lastFlushTimestamp + flushConsumeQueueThoroughInterval)) { + this.lastFlushTimestamp = currentTimeMillis; + flushConsumeQueueLeastPages = 0; + logicsMsgTimestamp = DefaultMessageStore.this.getStoreCheckpoint().getLogicsMsgTimestamp(); + } + + ConcurrentHashMap> tables = + DefaultMessageStore.this.consumeQueueTable; + + for (ConcurrentHashMap maps : tables.values()) { + for (ConsumeQueue cq : maps.values()) { + boolean result = false; + for (int i = 0; i < retryTimes && !result; i++) { + result = cq.commit(flushConsumeQueueLeastPages); + } + } + } + + if (0 == flushConsumeQueueLeastPages) { + if (logicsMsgTimestamp > 0) { + DefaultMessageStore.this.getStoreCheckpoint().setLogicsMsgTimestamp(logicsMsgTimestamp); + } + DefaultMessageStore.this.getStoreCheckpoint().flush(); + } + } + + + public void run() { + DefaultMessageStore.log.info(this.getServiceName() + " service started"); + + while (!this.isStoped()) { + try { + int interval = + DefaultMessageStore.this.getMessageStoreConfig().getFlushIntervalConsumeQueue(); + this.waitForRunning(interval); + this.doFlush(1); + } + catch (Exception e) { + DefaultMessageStore.log.warn(this.getServiceName() + " service has exception. ", e); + } + } + + // 正常shutdown时，要保证全部刷盘才退出 + this.doFlush(RetryTimesOver); + + DefaultMessageStore.log.info(this.getServiceName() + " service end"); + } + + + @Override + public String getServiceName() { + return FlushConsumeQueueService.class.getSimpleName(); + } + + + @Override + public long getJointime() { + return 1000 * 60; + } + } + + /** + * 分发消息索引服务 + */ + class DispatchMessageService extends ServiceThread { + private volatile List requestsWrite; + private volatile List requestsRead; + + + public DispatchMessageService(int putMsgIndexHightWater) { + putMsgIndexHightWater *= 1.5; + this.requestsWrite = new ArrayList(putMsgIndexHightWater); + this.requestsRead = new ArrayList(putMsgIndexHightWater); + } + + + public boolean hasRemainMessage() { + List reqs = this.requestsWrite; + if (reqs != null && !reqs.isEmpty()) { + return true; + } + + reqs = this.requestsRead; + if (reqs != null && !reqs.isEmpty()) { + return true; + } + + return false; + } + + + public void putRequest(final DispatchRequest dispatchRequest) { + int requestsWriteSize = 0; + int putMsgIndexHightWater = + DefaultMessageStore.this.getMessageStoreConfig().getPutMsgIndexHightWater(); + synchronized (this) { + this.requestsWrite.add(dispatchRequest); + requestsWriteSize = this.requestsWrite.size(); + if (!this.hasNotified) { + this.hasNotified = true; + this.notify(); + } + } + + DefaultMessageStore.this.getStoreStatsService().setDispatchMaxBuffer(requestsWriteSize); + + // 这里主动做流控，防止CommitLog写入太快，导致消费队列被冲垮 + if (requestsWriteSize > putMsgIndexHightWater) { + try { + if (log.isDebugEnabled()) { + log.debug("Message index buffer size " + requestsWriteSize + " > high water " + + putMsgIndexHightWater); + } + + Thread.sleep(1); + } + catch (InterruptedException e) { + } + } + } + + + private void swapRequests() { + List tmp = this.requestsWrite; + this.requestsWrite = this.requestsRead; + this.requestsRead = tmp; + } + + + private void doDispatch() { + if (!this.requestsRead.isEmpty()) { + for (DispatchRequest req : this.requestsRead) { + + final int tranType = MessageSysFlag.getTransactionValue(req.getSysFlag()); + // 1、分发消息位置信息到ConsumeQueue + switch (tranType) { + case MessageSysFlag.TransactionNotType: + case MessageSysFlag.TransactionCommitType: + // 将请求发到具体的Consume Queue + DefaultMessageStore.this.putMessagePostionInfo(req.getTopic(), req.getQueueId(), + req.getCommitLogOffset(), req.getMsgSize(), req.getTagsCode(), + req.getStoreTimestamp(), req.getConsumeQueueOffset()); + break; + case MessageSysFlag.TransactionPreparedType: + case MessageSysFlag.TransactionRollbackType: + break; + } + } + + if (DefaultMessageStore.this.getMessageStoreConfig().isMessageIndexEnable()) { + DefaultMessageStore.this.indexService.putRequest(this.requestsRead.toArray()); + } + + this.requestsRead.clear(); + } + } + + + public void run() { + DefaultMessageStore.log.info(this.getServiceName() + " service started"); + + while (!this.isStoped()) { + try { + this.waitForRunning(0); + this.doDispatch(); + } + catch (Exception e) { + DefaultMessageStore.log.warn(this.getServiceName() + " service has exception. ", e); + } + } + + // 在正常shutdown情况下，要保证所有消息都dispatch + try { + Thread.sleep(5 * 1000); + } + catch (InterruptedException e) { + DefaultMessageStore.log.warn("DispatchMessageService Exception, ", e); + } + + synchronized (this) { + this.swapRequests(); + } + + this.doDispatch(); + + DefaultMessageStore.log.info(this.getServiceName() + " service end"); + } + + + @Override + protected void onWaitEnd() { + this.swapRequests(); + } + + + @Override + public String getServiceName() { + return DispatchMessageService.class.getSimpleName(); + } + } + + /** + * SLAVE: 从物理队列Load消息，并分发到各个逻辑队列 + */ + class ReputMessageService extends ServiceThread { + // 从这里开始解析物理队列数据，并分发到逻辑队列 + private volatile long reputFromOffset = 0; + + + public long getReputFromOffset() { + return reputFromOffset; + } + + + public void setReputFromOffset(long reputFromOffset) { + this.reputFromOffset = reputFromOffset; + } + + + private void doReput() { + for (boolean doNext = true; doNext;) { + SelectMapedBufferResult result = DefaultMessageStore.this.commitLog.getData(reputFromOffset); + if (result != null) { + try { + for (int readSize = 0; readSize < result.getSize() && doNext;) { + DispatchRequest dispatchRequest = + DefaultMessageStore.this.commitLog.checkMessageAndReturnSize( + result.getByteBuffer(), false, false); + int size = dispatchRequest.getMsgSize(); + // 正常数据 + if (size > 0) { + DefaultMessageStore.this.putDispatchRequest(dispatchRequest); + + this.reputFromOffset = result.getStartOffset() + size; + readSize += size; + DefaultMessageStore.this.storeStatsService + .getSinglePutMessageTopicTimesTotal(dispatchRequest.getTopic()) + .incrementAndGet(); + DefaultMessageStore.this.storeStatsService.getSinglePutMessageTopicSizeTotal( + dispatchRequest.getTopic()).addAndGet(dispatchRequest.getMsgSize()); + } + // 文件中间读到错误 + else if (size == -1) { + doNext = false; + } + // 走到文件末尾，切换至下一个文件 + else if (size == 0) { + this.reputFromOffset = + DefaultMessageStore.this.commitLog.rollNextFile(this.reputFromOffset); + readSize = result.getSize(); + } + } + } + finally { + result.release(); + } + } + else { + doNext = false; + } + } + } + + + @Override + public void run() { + DefaultMessageStore.log.info(this.getServiceName() + " service started"); + + while (!this.isStoped()) { + try { + this.waitForRunning(1000); + this.doReput(); + } + catch (Exception e) { + DefaultMessageStore.log.warn(this.getServiceName() + " service has exception. ", e); + } + } + + DefaultMessageStore.log.info(this.getServiceName() + " service end"); + } + + + @Override + public String getServiceName() { + return ReputMessageService.class.getSimpleName(); + } + + } + + + @Override + public long getCommitLogOffsetInQueue(String topic, int queueId, long cqOffset) { + ConsumeQueue consumeQueue = findConsumeQueue(topic, queueId); + if (consumeQueue != null) { + SelectMapedBufferResult bufferConsumeQueue = consumeQueue.getIndexBuffer(cqOffset); + if (bufferConsumeQueue != null) { + try { + long offsetPy = bufferConsumeQueue.getByteBuffer().getLong(); + return offsetPy; + } + finally { + bufferConsumeQueue.release(); + } + } + } + + return 0; + } + + + @Override + public long getMinPhyOffset() { + return this.commitLog.getMinOffset(); + } + + + @Override + public long slaveFallBehindMuch() { + return this.commitLog.getMaxOffset() - this.haService.getPush2SlaveMaxOffset().get(); + } + + + @Override + public int cleanUnusedTopic(Set topics) { + Iterator>> it = + this.consumeQueueTable.entrySet().iterator(); + while (it.hasNext()) { + Entry> next = it.next(); + String topic = next.getKey(); + // Topic可以删除 + if (!topics.contains(topic) && !topic.equals(ScheduleMessageService.SCHEDULE_TOPIC)) { + ConcurrentHashMap queueTable = next.getValue(); + for (ConsumeQueue cq : queueTable.values()) { + cq.destroy(); + log.info("cleanUnusedTopic: {} {} ConsumeQueue cleaned",// + cq.getTopic(), // + cq.getQueueId() // + ); + + this.commitLog.removeQueurFromTopicQueueTable(cq.getTopic(), cq.getQueueId()); + } + it.remove(); + + log.info("cleanUnusedTopic: {},topic destroyed", topic); + } + } + + return 0; + } + + + public Map getMessageIds(final String topic, final int queueId, long minOffset, + long maxOffset, SocketAddress storeHost) { + Map messageIds = new HashMap(); + if (this.shutdown) { + return messageIds; + } + + ConsumeQueue consumeQueue = findConsumeQueue(topic, queueId); + if (consumeQueue != null) { + minOffset = Math.max(minOffset, consumeQueue.getMinOffsetInQuque()); + maxOffset = Math.min(maxOffset, consumeQueue.getMaxOffsetInQuque()); + + if (maxOffset == 0) { + return messageIds; + } + + long nextOffset = minOffset; + while (nextOffset < maxOffset) { + SelectMapedBufferResult bufferConsumeQueue = consumeQueue.getIndexBuffer(nextOffset); + if (bufferConsumeQueue != null) { + try { + int i = 0; + for (; i < bufferConsumeQueue.getSize(); i += ConsumeQueue.CQStoreUnitSize) { + long offsetPy = bufferConsumeQueue.getByteBuffer().getLong(); + final ByteBuffer msgIdMemory = ByteBuffer.allocate(MessageDecoder.MSG_ID_LENGTH); + String msgId = + MessageDecoder.createMessageId(msgIdMemory, + MessageExt.SocketAddress2ByteBuffer(storeHost), offsetPy); + messageIds.put(msgId, nextOffset++); + if (nextOffset > maxOffset) { + return messageIds; + } + } + } + finally { + // 必须释放资源 + bufferConsumeQueue.release(); + } + } + else { + return messageIds; + } + } + } + return messageIds; + } + + + private boolean checkInDiskByCommitOffset(long offsetPy) { + long maxOffsetPy = this.commitLog.getMaxOffset(); + long memory = + (long) (StoreUtil.TotalPhysicalMemorySize * (this.messageStoreConfig + .getAccessMessageInMemoryMaxRatio() / 100.0)); + return maxOffsetPy - offsetPy > memory; + } + + + @Override + public boolean checkInDiskByConsumeOffset(final String topic, final int queueId, long consumeOffset) { + ConsumeQueue consumeQueue = findConsumeQueue(topic, queueId); + if (consumeQueue != null) { + SelectMapedBufferResult bufferConsumeQueue = consumeQueue.getIndexBuffer(consumeOffset); + if (bufferConsumeQueue != null) { + try { + int i = 0; + for (; i < bufferConsumeQueue.getSize(); i += ConsumeQueue.CQStoreUnitSize) { + long offsetPy = bufferConsumeQueue.getByteBuffer().getLong(); + return checkInDiskByCommitOffset(offsetPy); + } + } + finally { + // 必须释放资源 + bufferConsumeQueue.release(); + } + } + else { + return false; + } + } + return false; + } +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/DispatchRequest.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/DispatchRequest.java new file mode 100644 index 000000000..c55562e67 --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/DispatchRequest.java @@ -0,0 +1,146 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store; + +/** + * 分发消息位置信息到逻辑队列和索引服务 + * + * @author shijia.wxr + * @since 2013-7-21 + */ +public class DispatchRequest { + private final String topic; + private final int queueId; + private final long commitLogOffset; + private final int msgSize; + private final long tagsCode; + private final long storeTimestamp; + private final long consumeQueueOffset; + private final String keys; + /** + * 事务相关部分 + */ + private final int sysFlag; + private final long preparedTransactionOffset; + + + public DispatchRequest(// + final String topic,// 1 + final int queueId,// 2 + final long commitLogOffset,// 3 + final int msgSize,// 4 + final long tagsCode,// 5 + final long storeTimestamp,// 6 + final long consumeQueueOffset,// 7 + final String keys,// 8 + /** + * 事务相关部分 + */ + final int sysFlag,// 9 + final long preparedTransactionOffset// 10 + ) { + this.topic = topic; + this.queueId = queueId; + this.commitLogOffset = commitLogOffset; + this.msgSize = msgSize; + this.tagsCode = tagsCode; + this.storeTimestamp = storeTimestamp; + this.consumeQueueOffset = consumeQueueOffset; + this.keys = keys; + + /** + * 事务相关部分 + */ + this.sysFlag = sysFlag; + this.preparedTransactionOffset = preparedTransactionOffset; + } + + + public DispatchRequest(int size) { + // 1 + this.topic = ""; + // 2 + this.queueId = 0; + // 3 + this.commitLogOffset = 0; + // 4 + this.msgSize = size; + // 5 + this.tagsCode = 0; + // 6 + this.storeTimestamp = 0; + // 7 + this.consumeQueueOffset = 0; + // 8 + this.keys = ""; + + /** + * 事务相关部分 + */ + this.sysFlag = 0; + this.preparedTransactionOffset = 0; + } + + + public String getTopic() { + return topic; + } + + + public int getQueueId() { + return queueId; + } + + + public long getCommitLogOffset() { + return commitLogOffset; + } + + + public int getMsgSize() { + return msgSize; + } + + + public long getStoreTimestamp() { + return storeTimestamp; + } + + + public long getConsumeQueueOffset() { + return consumeQueueOffset; + } + + + public String getKeys() { + return keys; + } + + + public long getTagsCode() { + return tagsCode; + } + + + public int getSysFlag() { + return sysFlag; + } + + + public long getPreparedTransactionOffset() { + return preparedTransactionOffset; + } +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/GetMessageResult.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/GetMessageResult.java new file mode 100644 index 000000000..4563d9710 --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/GetMessageResult.java @@ -0,0 +1,149 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + + +/** + * 访问消息返回结果 + * + * @author shijia.wxr + * @since 2013-7-21 + */ +public class GetMessageResult { + // 多个连续的消息集合 + private final List messageMapedList = + new ArrayList(100); + // 用来向Consumer传送消息 + private final List messageBufferList = new ArrayList(100); + // 枚举变量，取消息结果 + private GetMessageStatus status; + // 当被过滤后，返回下一次开始的Offset + private long nextBeginOffset; + // 逻辑队列中的最小Offset + private long minOffset; + // 逻辑队列中的最大Offset + private long maxOffset; + // ByteBuffer 总字节数 + private int bufferTotalSize = 0; + // 是否建议从slave拉消息 + private boolean suggestPullingFromSlave = false; + + + public GetMessageResult() { + } + + + public GetMessageStatus getStatus() { + return status; + } + + + public void setStatus(GetMessageStatus status) { + this.status = status; + } + + + public long getNextBeginOffset() { + return nextBeginOffset; + } + + + public void setNextBeginOffset(long nextBeginOffset) { + this.nextBeginOffset = nextBeginOffset; + } + + + public long getMinOffset() { + return minOffset; + } + + + public void setMinOffset(long minOffset) { + this.minOffset = minOffset; + } + + + public long getMaxOffset() { + return maxOffset; + } + + + public void setMaxOffset(long maxOffset) { + this.maxOffset = maxOffset; + } + + + public List getMessageMapedList() { + return messageMapedList; + } + + + public List getMessageBufferList() { + return messageBufferList; + } + + + public void addMessage(final SelectMapedBufferResult mapedBuffer) { + this.messageMapedList.add(mapedBuffer); + this.messageBufferList.add(mapedBuffer.getByteBuffer()); + this.bufferTotalSize += mapedBuffer.getSize(); + } + + + public void release() { + for (SelectMapedBufferResult select : this.messageMapedList) { + select.release(); + } + } + + + public int getBufferTotalSize() { + return bufferTotalSize; + } + + + public void setBufferTotalSize(int bufferTotalSize) { + this.bufferTotalSize = bufferTotalSize; + } + + + public int getMessageCount() { + return this.messageMapedList.size(); + } + + + public boolean isSuggestPullingFromSlave() { + return suggestPullingFromSlave; + } + + + public void setSuggestPullingFromSlave(boolean suggestPullingFromSlave) { + this.suggestPullingFromSlave = suggestPullingFromSlave; + } + + + @Override + public String toString() { + return "GetMessageResult [status=" + status + ", nextBeginOffset=" + nextBeginOffset + ", minOffset=" + + minOffset + ", maxOffset=" + maxOffset + ", bufferTotalSize=" + bufferTotalSize + + ", suggestPullingFromSlave=" + suggestPullingFromSlave + "]"; + } + +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/GetMessageStatus.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/GetMessageStatus.java new file mode 100644 index 000000000..7aa885de2 --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/GetMessageStatus.java @@ -0,0 +1,43 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store; + +/** + * 访问消息返回的状态码 + * + * @author shijia.wxr + * @since 2013-7-21 + */ +public enum GetMessageStatus { + // 找到消息 + FOUND, + // offset正确，但是过滤后没有匹配的消息 + NO_MATCHED_MESSAGE, + // offset正确，但是物理队列消息正在被删除 + MESSAGE_WAS_REMOVING, + // offset正确，但是从逻辑队列没有找到，可能正在被删除 + OFFSET_FOUND_NULL, + // offset错误，严重溢出 + OFFSET_OVERFLOW_BADLY, + // offset错误，溢出1个 + OFFSET_OVERFLOW_ONE, + // offset错误，太小了 + OFFSET_TOO_SMALL, + // 没有对应的逻辑队列 + NO_MATCHED_LOGIC_QUEUE, + // 队列中一条消息都没有 + NO_MESSAGE_IN_QUEUE, +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/MapedFile.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/MapedFile.java new file mode 100644 index 000000000..398df93e7 --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/MapedFile.java @@ -0,0 +1,466 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.lang.reflect.Method; +import java.nio.ByteBuffer; +import java.nio.MappedByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.channels.FileChannel.MapMode; +import java.security.AccessController; +import java.security.PrivilegedAction; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.constant.LoggerName; + + +/** + * Pagecache文件访问封装 + * + * @author shijia.wxr + * @since 2013-7-21 + */ +public class MapedFile extends ReferenceResource { + public static final int OS_PAGE_SIZE = 1024 * 4; + private static final Logger log = LoggerFactory.getLogger(LoggerName.StoreLoggerName); + // 当前JVM中映射的虚拟内存总大小 + private static final AtomicLong TotalMapedVitualMemory = new AtomicLong(0); + // 当前JVM中mmap句柄数量 + private static final AtomicInteger TotalMapedFiles = new AtomicInteger(0); + // 映射的文件名 + private final String fileName; + // 映射的起始偏移量 + private final long fileFromOffset; + // 映射的文件大小，定长 + private final int fileSize; + // 映射的文件 + private final File file; + // 映射的内存对象，position永远不变 + private final MappedByteBuffer mappedByteBuffer; + // 当前写到什么位置 + private final AtomicInteger wrotePostion = new AtomicInteger(0); + // Flush到什么位置 + private final AtomicInteger committedPosition = new AtomicInteger(0); + // 映射的FileChannel对象 + private FileChannel fileChannel; + // 最后一条消息存储时间 + private volatile long storeTimestamp = 0; + private boolean firstCreateInQueue = false; + + + public MapedFile(final String fileName, final int fileSize) throws IOException { + this.fileName = fileName; + this.fileSize = fileSize; + this.file = new File(fileName); + this.fileFromOffset = Long.parseLong(this.file.getName()); + boolean ok = false; + + ensureDirOK(this.file.getParent()); + + try { + this.fileChannel = new RandomAccessFile(this.file, "rw").getChannel(); + this.mappedByteBuffer = this.fileChannel.map(MapMode.READ_WRITE, 0, fileSize); + TotalMapedVitualMemory.addAndGet(fileSize); + TotalMapedFiles.incrementAndGet(); + ok = true; + } + catch (FileNotFoundException e) { + log.error("create file channel " + this.fileName + " Failed. ", e); + throw e; + } + catch (IOException e) { + log.error("map file " + this.fileName + " Failed. ", e); + throw e; + } + finally { + if (!ok && this.fileChannel != null) { + this.fileChannel.close(); + } + } + } + + + public static void ensureDirOK(final String dirName) { + if (dirName != null) { + File f = new File(dirName); + if (!f.exists()) { + boolean result = f.mkdirs(); + log.info(dirName + " mkdir " + (result ? "OK" : "Failed")); + } + } + } + + + public static void clean(final ByteBuffer buffer) { + if (buffer == null || !buffer.isDirect() || buffer.capacity() == 0) + return; + invoke(invoke(viewed(buffer), "cleaner"), "clean"); + } + + + private static Object invoke(final Object target, final String methodName, final Class... args) { + return AccessController.doPrivileged(new PrivilegedAction() { + public Object run() { + try { + Method method = method(target, methodName, args); + method.setAccessible(true); + return method.invoke(target); + } + catch (Exception e) { + throw new IllegalStateException(e); + } + } + }); + } + + + private static Method method(Object target, String methodName, Class[] args) + throws NoSuchMethodException { + try { + return target.getClass().getMethod(methodName, args); + } + catch (NoSuchMethodException e) { + return target.getClass().getDeclaredMethod(methodName, args); + } + } + + + private static ByteBuffer viewed(ByteBuffer buffer) { + String methodName = "viewedBuffer"; + + // JDK7中将DirectByteBuffer类中的viewedBuffer方法换成了attachment方法 + Method[] methods = buffer.getClass().getMethods(); + for (int i = 0; i < methods.length; i++) { + if (methods[i].getName().equals("attachment")) { + methodName = "attachment"; + break; + } + } + + ByteBuffer viewedBuffer = (ByteBuffer) invoke(buffer, methodName); + if (viewedBuffer == null) + return buffer; + else + return viewed(viewedBuffer); + } + + + public static int getTotalmapedfiles() { + return TotalMapedFiles.get(); + } + + + public static long getTotalMapedVitualMemory() { + return TotalMapedVitualMemory.get(); + } + + + public long getLastModifiedTimestamp() { + return this.file.lastModified(); + } + + + public String getFileName() { + return fileName; + } + + + /** + * 获取文件大小 + */ + public int getFileSize() { + return fileSize; + } + + + public FileChannel getFileChannel() { + return fileChannel; + } + + + /** + * 向MapedBuffer追加消息
+ * + * @param msg + * 要追加的消息 + * @param cb + * 用来对消息进行序列化，尤其对于依赖MapedFile Offset的属性进行动态序列化 + * @return 是否成功，写入多少数据 + */ + public AppendMessageResult appendMessage(final Object msg, final AppendMessageCallback cb) { + assert msg != null; + assert cb != null; + + int currentPos = this.wrotePostion.get(); + + // 表示有空余空间 + if (currentPos < this.fileSize) { + ByteBuffer byteBuffer = this.mappedByteBuffer.slice(); + byteBuffer.position(currentPos); + AppendMessageResult result = + cb.doAppend(this.getFileFromOffset(), byteBuffer, this.fileSize - currentPos, msg); + this.wrotePostion.addAndGet(result.getWroteBytes()); + this.storeTimestamp = result.getStoreTimestamp(); + return result; + } + + // 上层应用应该保证不会走到这里 + log.error("MapedFile.appendMessage return null, wrotePostion: " + currentPos + " fileSize: " + + this.fileSize); + return new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR); + } + + + /** + * 文件起始偏移量 + */ + public long getFileFromOffset() { + return this.fileFromOffset; + } + + + /** + * 向存储层追加数据，一般在SLAVE存储结构中使用 + * + * @return 返回写入了多少数据 + */ + public boolean appendMessage(final byte[] data) { + int currentPos = this.wrotePostion.get(); + + // 表示有空余空间 + if ((currentPos + data.length) <= this.fileSize) { + ByteBuffer byteBuffer = this.mappedByteBuffer.slice(); + byteBuffer.position(currentPos); + byteBuffer.put(data); + this.wrotePostion.addAndGet(data.length); + return true; + } + + return false; + } + + + /** + * 消息刷盘 + * + * @param flushLeastPages + * 至少刷几个page + * @return + */ + public int commit(final int flushLeastPages) { + if (this.isAbleToFlush(flushLeastPages)) { + if (this.hold()) { + int value = this.wrotePostion.get(); + this.mappedByteBuffer.force(); + this.committedPosition.set(value); + this.release(); + } + else { + log.warn("in commit, hold failed, commit offset = " + this.committedPosition.get()); + this.committedPosition.set(this.wrotePostion.get()); + } + } + + return this.getCommittedPosition(); + } + + + public int getCommittedPosition() { + return committedPosition.get(); + } + + + public void setCommittedPosition(int pos) { + this.committedPosition.set(pos); + } + + + private boolean isAbleToFlush(final int flushLeastPages) { + int flush = this.committedPosition.get(); + int write = this.wrotePostion.get(); + + // 如果当前文件已经写满，应该立刻刷盘 + if (this.isFull()) { + return true; + } + + // 只有未刷盘数据满足指定page数目才刷盘 + if (flushLeastPages > 0) { + return ((write / OS_PAGE_SIZE) - (flush / OS_PAGE_SIZE)) >= flushLeastPages; + } + + return write > flush; + } + + + public boolean isFull() { + return this.fileSize == this.wrotePostion.get(); + } + + + public SelectMapedBufferResult selectMapedBuffer(int pos, int size) { + // 有消息 + if ((pos + size) <= this.wrotePostion.get()) { + // 从MapedBuffer读 + if (this.hold()) { + ByteBuffer byteBuffer = this.mappedByteBuffer.slice(); + byteBuffer.position(pos); + ByteBuffer byteBufferNew = byteBuffer.slice(); + byteBufferNew.limit(size); + return new SelectMapedBufferResult(this.fileFromOffset + pos, byteBufferNew, size, this); + } + else { + log.warn("matched, but hold failed, request pos: " + pos + ", fileFromOffset: " + + this.fileFromOffset); + } + } + // 请求参数非法 + else { + log.warn("selectMapedBuffer request pos invalid, request pos: " + pos + ", size: " + size + + ", fileFromOffset: " + this.fileFromOffset); + } + + // 非法参数或者mmap资源已经被释放 + return null; + } + + + /** + * 读逻辑分区 + */ + public SelectMapedBufferResult selectMapedBuffer(int pos) { + if (pos < this.wrotePostion.get() && pos >= 0) { + if (this.hold()) { + ByteBuffer byteBuffer = this.mappedByteBuffer.slice(); + byteBuffer.position(pos); + int size = this.wrotePostion.get() - pos; + ByteBuffer byteBufferNew = byteBuffer.slice(); + byteBufferNew.limit(size); + return new SelectMapedBufferResult(this.fileFromOffset + pos, byteBufferNew, size, this); + } + } + + // 非法参数或者mmap资源已经被释放 + return null; + } + + + @Override + public boolean cleanup(final long currentRef) { + // 如果没有被shutdown，则不可以unmap文件，否则会crash + if (this.isAvailable()) { + log.error("this file[REF:" + currentRef + "] " + this.fileName + + " have not shutdown, stop unmaping."); + return false; + } + + // 如果已经cleanup，再次操作会引起crash + if (this.isCleanupOver()) { + log.error("this file[REF:" + currentRef + "] " + this.fileName + + " have cleanup, do not do it again."); + // 必须返回true + return true; + } + + clean(this.mappedByteBuffer); + TotalMapedVitualMemory.addAndGet(this.fileSize * (-1)); + TotalMapedFiles.decrementAndGet(); + log.info("unmap file[REF:" + currentRef + "] " + this.fileName + " OK"); + return true; + } + + + /** + * 清理资源，destroy与调用shutdown的线程必须是同一个 + * + * @return 是否被destory成功，上层调用需要对失败情况处理，失败后尝试重试 + */ + public boolean destroy(final long intervalForcibly) { + this.shutdown(intervalForcibly); + + if (this.isCleanupOver()) { + try { + this.fileChannel.close(); + log.info("close file channel " + this.fileName + " OK"); + + long beginTime = System.currentTimeMillis(); + boolean result = this.file.delete(); + log.info("delete file[REF:" + this.getRefCount() + "] " + this.fileName + + (result ? " OK, " : " Failed, ") + "W:" + this.getWrotePostion() + " M:" + + this.getCommittedPosition() + ", " + + UtilAll.computeEclipseTimeMilliseconds(beginTime)); + } + catch (Exception e) { + log.warn("close file channel " + this.fileName + " Failed. ", e); + } + + return true; + } + else { + log.warn("destroy maped file[REF:" + this.getRefCount() + "] " + this.fileName + + " Failed. cleanupOver: " + this.cleanupOver); + } + + return false; + } + + + public int getWrotePostion() { + return wrotePostion.get(); + } + + + public void setWrotePostion(int pos) { + this.wrotePostion.set(pos); + } + + + public MappedByteBuffer getMappedByteBuffer() { + return mappedByteBuffer; + } + + + /** + * 方法不能在运行时调用，不安全。只在启动时，reload已有数据时调用 + */ + public ByteBuffer sliceByteBuffer() { + return this.mappedByteBuffer.slice(); + } + + + public long getStoreTimestamp() { + return storeTimestamp; + } + + + public boolean isFirstCreateInQueue() { + return firstCreateInQueue; + } + + + public void setFirstCreateInQueue(boolean firstCreateInQueue) { + this.firstCreateInQueue = firstCreateInQueue; + } +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/MapedFileQueue.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/MapedFileQueue.java new file mode 100644 index 000000000..3e067becb --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/MapedFileQueue.java @@ -0,0 +1,626 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.constant.LoggerName; + + +/** + * 存储队列，数据定时删除，无限增长
+ * 队列是由多个文件组成 + * + * @author shijia.wxr + * @since 2013-7-21 + */ +public class MapedFileQueue { + private static final Logger log = LoggerFactory.getLogger(LoggerName.StoreLoggerName); + private static final Logger logError = LoggerFactory.getLogger(LoggerName.StoreErrorLoggerName); + // 每次触发删除文件，最多删除多少个文件 + private static final int DeleteFilesBatchMax = 10; + // 文件存储位置 + private final String storePath; + // 每个文件的大小 + private final int mapedFileSize; + // 各个文件 + private final List mapedFiles = new ArrayList(); + // 读写锁（针对mapedFiles） + private final ReadWriteLock readWriteLock = new ReentrantReadWriteLock(); + // 预分配MapedFile对象服务 + private final AllocateMapedFileService allocateMapedFileService; + // 刷盘刷到哪里 + private long committedWhere = 0; + // 最后一条消息存储时间 + private volatile long storeTimestamp = 0; + + + public MapedFileQueue(final String storePath, int mapedFileSize, + AllocateMapedFileService allocateMapedFileService) { + this.storePath = storePath; + this.mapedFileSize = mapedFileSize; + this.allocateMapedFileService = allocateMapedFileService; + } + + + public MapedFile getMapedFileByTime(final long timestamp) { + Object[] mfs = this.copyMapedFiles(0); + + if (null == mfs) + return null; + + for (int i = 0; i < mfs.length; i++) { + MapedFile mapedFile = (MapedFile) mfs[i]; + if (mapedFile.getLastModifiedTimestamp() >= timestamp) { + return mapedFile; + } + } + + return (MapedFile) mfs[mfs.length - 1]; + } + + + private Object[] copyMapedFiles(final int reservedMapedFiles) { + Object[] mfs = null; + + try { + this.readWriteLock.readLock().lock(); + if (this.mapedFiles.size() <= reservedMapedFiles) { + return null; + } + + mfs = this.mapedFiles.toArray(); + } + catch (Exception e) { + e.printStackTrace(); + } + finally { + this.readWriteLock.readLock().unlock(); + } + return mfs; + } + + + /** + * recover时调用，不需要加锁 + */ + public void truncateDirtyFiles(long offset) { + List willRemoveFiles = new ArrayList(); + + for (MapedFile file : this.mapedFiles) { + long fileTailOffset = file.getFileFromOffset() + this.mapedFileSize; + if (fileTailOffset > offset) { + if (offset >= file.getFileFromOffset()) { + file.setWrotePostion((int) (offset % this.mapedFileSize)); + file.setCommittedPosition((int) (offset % this.mapedFileSize)); + } + else { + // 将文件删除掉 + file.destroy(1000); + willRemoveFiles.add(file); + } + } + } + + this.deleteExpiredFile(willRemoveFiles); + } + + + /** + * 删除文件只能从头开始删 + */ + private void deleteExpiredFile(List files) { + if (!files.isEmpty()) { + try { + this.readWriteLock.writeLock().lock(); + for (MapedFile file : files) { + if (!this.mapedFiles.remove(file)) { + log.error("deleteExpiredFile remove failed."); + break; + } + } + } + catch (Exception e) { + log.error("deleteExpiredFile has exception.", e); + } + finally { + this.readWriteLock.writeLock().unlock(); + } + } + } + + + public boolean load() { + File dir = new File(this.storePath); + File[] files = dir.listFiles(); + if (files != null) { + // ascending order + Arrays.sort(files); + for (File file : files) { + // 校验文件大小是否匹配 + if (file.length() != this.mapedFileSize) { + log.warn(file + "\t" + file.length() + + " length not matched message store config value, ignore it"); + return true; + } + + // 恢复队列 + try { + MapedFile mapedFile = new MapedFile(file.getPath(), mapedFileSize); + + mapedFile.setWrotePostion(this.mapedFileSize); + mapedFile.setCommittedPosition(this.mapedFileSize); + this.mapedFiles.add(mapedFile); + log.info("load " + file.getPath() + " OK"); + } + catch (IOException e) { + log.error("load file " + file + " error", e); + return false; + } + } + } + + return true; + } + + + /** + * 刷盘进度落后了多少 + */ + public long howMuchFallBehind() { + if (this.mapedFiles.isEmpty()) + return 0; + + long committed = this.committedWhere; + if (committed != 0) { + MapedFile mapedFile = this.getLastMapedFile(); + if (mapedFile != null) { + return (mapedFile.getFileFromOffset() + mapedFile.getWrotePostion()) - committed; + } + } + + return 0; + } + + + public MapedFile getLastMapedFile() { + return this.getLastMapedFile(0); + } + + + /** + * 获取最后一个MapedFile对象，如果一个都没有，则新创建一个，如果最后一个写满了，则新创建一个 + * + * @param startOffset + * 如果创建新的文件，起始offset + * @return + */ + public MapedFile getLastMapedFile(final long startOffset) { + long createOffset = -1; + MapedFile mapedFileLast = null; + { + this.readWriteLock.readLock().lock(); + if (this.mapedFiles.isEmpty()) { + createOffset = startOffset - (startOffset % this.mapedFileSize); + } + else { + mapedFileLast = this.mapedFiles.get(this.mapedFiles.size() - 1); + } + this.readWriteLock.readLock().unlock(); + } + + if (mapedFileLast != null && mapedFileLast.isFull()) { + createOffset = mapedFileLast.getFileFromOffset() + this.mapedFileSize; + } + + if (createOffset != -1) { + String nextFilePath = this.storePath + File.separator + UtilAll.offset2FileName(createOffset); + String nextNextFilePath = + this.storePath + File.separator + + UtilAll.offset2FileName(createOffset + this.mapedFileSize); + MapedFile mapedFile = null; + + if (this.allocateMapedFileService != null) { + mapedFile = + this.allocateMapedFileService.putRequestAndReturnMapedFile(nextFilePath, + nextNextFilePath, this.mapedFileSize); + } + else { + try { + mapedFile = new MapedFile(nextFilePath, this.mapedFileSize); + } + catch (IOException e) { + log.error("create mapedfile exception", e); + } + } + + if (mapedFile != null) { + this.readWriteLock.writeLock().lock(); + if (this.mapedFiles.isEmpty()) { + mapedFile.setFirstCreateInQueue(true); + } + this.mapedFiles.add(mapedFile); + this.readWriteLock.writeLock().unlock(); + } + + return mapedFile; + } + + return mapedFileLast; + } + + + /** + * 获取队列的最小Offset，如果队列为空，则返回-1 + */ + public long getMinOffset() { + try { + this.readWriteLock.readLock().lock(); + if (!this.mapedFiles.isEmpty()) { + return this.mapedFiles.get(0).getFileFromOffset(); + } + } + catch (Exception e) { + log.error("getMinOffset has exception.", e); + } + finally { + this.readWriteLock.readLock().unlock(); + } + + return -1; + } + + + public long getMaxOffset() { + try { + this.readWriteLock.readLock().lock(); + if (!this.mapedFiles.isEmpty()) { + int lastIndex = this.mapedFiles.size() - 1; + MapedFile mapedFile = this.mapedFiles.get(lastIndex); + return mapedFile.getFileFromOffset() + mapedFile.getWrotePostion(); + } + } + catch (Exception e) { + log.error("getMinOffset has exception.", e); + } + finally { + this.readWriteLock.readLock().unlock(); + } + + return 0; + } + + + /** + * 恢复时调用 + */ + public void deleteLastMapedFile() { + if (!this.mapedFiles.isEmpty()) { + int lastIndex = this.mapedFiles.size() - 1; + MapedFile mapedFile = this.mapedFiles.get(lastIndex); + mapedFile.destroy(1000); + this.mapedFiles.remove(mapedFile); + log.info("on recover, destroy a logic maped file " + mapedFile.getFileName()); + } + } + + + /** + * 根据文件过期时间来删除物理队列文件 + */ + public int deleteExpiredFileByTime(// + final long expiredTime, // + final int deleteFilesInterval, // + final long intervalForcibly,// + final boolean cleanImmediately// + ) { + Object[] mfs = this.copyMapedFiles(0); + + if (null == mfs) + return 0; + + // 最后一个文件处于写状态，不能删除 + int mfsLength = mfs.length - 1; + int deleteCount = 0; + List files = new ArrayList(); + if (null != mfs) { + for (int i = 0; i < mfsLength; i++) { + MapedFile mapedFile = (MapedFile) mfs[i]; + long liveMaxTimestamp = mapedFile.getLastModifiedTimestamp() + expiredTime; + if (System.currentTimeMillis() >= liveMaxTimestamp// + || cleanImmediately) { + if (mapedFile.destroy(intervalForcibly)) { + files.add(mapedFile); + deleteCount++; + + if (files.size() >= DeleteFilesBatchMax) { + break; + } + + if (deleteFilesInterval > 0 && (i + 1) < mfsLength) { + try { + Thread.sleep(deleteFilesInterval); + } + catch (InterruptedException e) { + } + } + } + else { + break; + } + } + } + } + + deleteExpiredFile(files); + + return deleteCount; + } + + + /** + * 根据物理队列最小Offset来删除逻辑队列 + * + * @param offset + * 物理队列最小offset + */ + public int deleteExpiredFileByOffset(long offset, int unitSize) { + Object[] mfs = this.copyMapedFiles(0); + + List files = new ArrayList(); + int deleteCount = 0; + if (null != mfs) { + // 最后一个文件处于写状态，不能删除 + int mfsLength = mfs.length - 1; + + // 这里遍历范围 0 ... last - 1 + for (int i = 0; i < mfsLength; i++) { + boolean destroy = true; + MapedFile mapedFile = (MapedFile) mfs[i]; + SelectMapedBufferResult result = mapedFile.selectMapedBuffer(this.mapedFileSize - unitSize); + if (result != null) { + long maxOffsetInLogicQueue = result.getByteBuffer().getLong(); + result.release(); + // 当前文件是否可以删除 + destroy = (maxOffsetInLogicQueue < offset); + if (destroy) { + log.info("physic min offset " + offset + ", logics in current mapedfile max offset " + + maxOffsetInLogicQueue + ", delete it"); + } + } + else { + log.warn("this being not excuted forever."); + break; + } + + if (destroy && mapedFile.destroy(1000 * 60)) { + files.add(mapedFile); + deleteCount++; + } + else { + break; + } + } + } + + deleteExpiredFile(files); + + return deleteCount; + } + + + /** + * 返回值表示是否全部刷盘完成 + * + * @return + */ + public boolean commit(final int flushLeastPages) { + boolean result = true; + MapedFile mapedFile = this.findMapedFileByOffset(this.committedWhere, true); + if (mapedFile != null) { + long tmpTimeStamp = mapedFile.getStoreTimestamp(); + int offset = mapedFile.commit(flushLeastPages); + long where = mapedFile.getFileFromOffset() + offset; + result = (where == this.committedWhere); + this.committedWhere = where; + if (0 == flushLeastPages) { + this.storeTimestamp = tmpTimeStamp; + } + } + + return result; + } + + + public MapedFile findMapedFileByOffset(final long offset, final boolean returnFirstOnNotFound) { + try { + this.readWriteLock.readLock().lock(); + MapedFile mapedFile = this.getFirstMapedFile(); + + if (mapedFile != null) { + int index = + (int) ((offset / this.mapedFileSize) - (mapedFile.getFileFromOffset() / this.mapedFileSize)); + if (index < 0 || index >= this.mapedFiles.size()) { + logError + .warn( + "findMapedFileByOffset offset not matched, request Offset: {}, index: {}, mapedFileSize: {}, mapedFiles count: {}, StackTrace: {}",// + offset,// + index,// + this.mapedFileSize,// + this.mapedFiles.size(),// + UtilAll.currentStackTrace()); + } + + try { + return this.mapedFiles.get(index); + } + catch (Exception e) { + if (returnFirstOnNotFound) { + return mapedFile; + } + } + } + } + catch (Exception e) { + log.error("findMapedFileByOffset Exception", e); + } + finally { + this.readWriteLock.readLock().unlock(); + } + + return null; + } + + + private MapedFile getFirstMapedFile() { + if (this.mapedFiles.isEmpty()) { + return null; + } + + return this.mapedFiles.get(0); + } + + + public MapedFile getLastMapedFile2() { + if (this.mapedFiles.isEmpty()) { + return null; + } + return this.mapedFiles.get(this.mapedFiles.size() - 1); + } + + + public MapedFile findMapedFileByOffset(final long offset) { + return findMapedFileByOffset(offset, false); + } + + + public long getMapedMemorySize() { + long size = 0; + + Object[] mfs = this.copyMapedFiles(0); + if (mfs != null) { + for (Object mf : mfs) { + if (((ReferenceResource) mf).isAvailable()) { + size += this.mapedFileSize; + } + } + } + + return size; + } + + + public boolean retryDeleteFirstFile(final long intervalForcibly) { + MapedFile mapedFile = this.getFirstMapedFileOnLock(); + if (mapedFile != null) { + if (!mapedFile.isAvailable()) { + log.warn("the mapedfile was destroyed once, but still alive, " + mapedFile.getFileName()); + boolean result = mapedFile.destroy(intervalForcibly); + if (result) { + log.warn("the mapedfile redelete OK, " + mapedFile.getFileName()); + List tmps = new ArrayList(); + tmps.add(mapedFile); + this.deleteExpiredFile(tmps); + } + else { + log.warn("the mapedfile redelete Failed, " + mapedFile.getFileName()); + } + + return result; + } + } + + return false; + } + + + public MapedFile getFirstMapedFileOnLock() { + try { + this.readWriteLock.readLock().lock(); + return this.getFirstMapedFile(); + } + finally { + this.readWriteLock.readLock().unlock(); + } + } + + + /** + * 关闭队列，队列数据还在，但是不能访问 + */ + public void shutdown(final long intervalForcibly) { + this.readWriteLock.readLock().lock(); + for (MapedFile mf : this.mapedFiles) { + mf.shutdown(intervalForcibly); + } + this.readWriteLock.readLock().unlock(); + } + + + /** + * 销毁队列，队列数据被删除，此函数有可能不成功 + */ + public void destroy() { + this.readWriteLock.writeLock().lock(); + for (MapedFile mf : this.mapedFiles) { + mf.destroy(1000 * 3); + } + this.mapedFiles.clear(); + this.committedWhere = 0; + + // delete parent directory + File file = new File(storePath); + if (file.isDirectory()) { + file.delete(); + } + this.readWriteLock.writeLock().unlock(); + } + + + public long getCommittedWhere() { + return committedWhere; + } + + + public void setCommittedWhere(long committedWhere) { + this.committedWhere = committedWhere; + } + + + public long getStoreTimestamp() { + return storeTimestamp; + } + + + public List getMapedFiles() { + return mapedFiles; + } + + + public int getMapedFileSize() { + return mapedFileSize; + } +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/MessageExtBrokerInner.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/MessageExtBrokerInner.java new file mode 100644 index 000000000..2e454d669 --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/MessageExtBrokerInner.java @@ -0,0 +1,63 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store; + +import com.alibaba.rocketmq.common.TopicFilterType; +import com.alibaba.rocketmq.common.message.MessageExt; + + +/** + * 存储内部使用的Message对象 + * + * @author shijia.wxr + * @since 2013-7-21 + */ +public class MessageExtBrokerInner extends MessageExt { + private static final long serialVersionUID = 7256001576878700634L; + private String propertiesString; + private long tagsCode; + + + /** + * 目前只支持单个标签的过滤 + */ + public static long tagsString2tagsCode(final TopicFilterType filter, final String tags) { + if (null == tags || tags.length() == 0) + return 0; + + return tags.hashCode(); + } + + + public String getPropertiesString() { + return propertiesString; + } + + + public void setPropertiesString(String propertiesString) { + this.propertiesString = propertiesString; + } + + + public long getTagsCode() { + return tagsCode; + } + + + public void setTagsCode(long tagsCode) { + this.tagsCode = tagsCode; + } +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/MessageFilter.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/MessageFilter.java new file mode 100644 index 000000000..e76fac258 --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/MessageFilter.java @@ -0,0 +1,29 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store; + +import com.alibaba.rocketmq.common.protocol.heartbeat.SubscriptionData; + + +/** + * 消息过滤接口 + * + * @author shijia.wxr + * @since 2013-7-21 + */ +public interface MessageFilter { + public boolean isMessageMatched(final SubscriptionData subscriptionData, final long tagsCode); +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/MessageStore.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/MessageStore.java new file mode 100644 index 000000000..d9f5246f2 --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/MessageStore.java @@ -0,0 +1,205 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store; + +import com.alibaba.rocketmq.common.message.MessageExt; +import com.alibaba.rocketmq.common.protocol.heartbeat.SubscriptionData; + +import java.net.SocketAddress; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + + +/** + * 存储层对外提供的接口 + * + * @author shijia.wxr + * @since 2013-7-21 + */ +public interface MessageStore { + + /** + * 重启时，加载数据 + */ + public boolean load(); + + + /** + * 启动服务 + */ + public void start() throws Exception; + + + /** + * 关闭服务 + */ + public void shutdown(); + + + /** + * 删除所有文件，单元测试会使用 + */ + public void destroy(); + + + /** + * 存储消息 + */ + public PutMessageResult putMessage(final MessageExtBrokerInner msg); + + + /** + * 读取消息，如果types为null，则不做过滤 + */ + public GetMessageResult getMessage(final String group, final String topic, final int queueId, + final long offset, final int maxMsgNums, final SubscriptionData subscriptionData); + + + /** + * 获取指定队列最大Offset 如果队列不存在，返回-1 + */ + public long getMaxOffsetInQuque(final String topic, final int queueId); + + + /** + * 获取指定队列最小Offset 如果队列不存在，返回-1 + */ + public long getMinOffsetInQuque(final String topic, final int queueId); + + + /** + * 获取消费队列记录的CommitLog Offset + */ + public long getCommitLogOffsetInQueue(final String topic, final int queueId, final long cqOffset); + + + /** + * 根据消息时间获取某个队列中对应的offset 1、如果指定时间（包含之前之后）有对应的消息，则获取距离此时间最近的offset（优先选择之前） + * 2、如果指定时间无对应消息，则返回0 + */ + public long getOffsetInQueueByTime(final String topic, final int queueId, final long timestamp); + + + /** + * 通过物理队列Offset，查询消息。 如果发生错误，则返回null + */ + public MessageExt lookMessageByOffset(final long commitLogOffset); + + + /** + * 通过物理队列Offset，查询消息。 如果发生错误，则返回null + */ + public SelectMapedBufferResult selectOneMessageByOffset(final long commitLogOffset); + + + public SelectMapedBufferResult selectOneMessageByOffset(final long commitLogOffset, final int msgSize); + + + /** + * 获取运行时统计数据 + */ + public String getRunningDataInfo(); + + + /** + * 获取运行时统计数据 + */ + public HashMap getRuntimeInfo(); + + + /** + * 获取物理队列最大offset + */ + public long getMaxPhyOffset(); + + + public long getMinPhyOffset(); + + + /** + * 获取队列中最早的消息时间 + */ + public long getEarliestMessageTime(final String topic, final int queueId); + + + public long getMessageStoreTimeStamp(final String topic, final int queueId, final long offset); + + + /** + * 获取队列中的消息总数 + */ + public long getMessageTotalInQueue(final String topic, final int queueId); + + + /** + * 数据复制使用：获取CommitLog数据 + */ + public SelectMapedBufferResult getCommitLogData(final long offset); + + + /** + * 数据复制使用：向CommitLog追加数据，并分发至各个Consume Queue + */ + public boolean appendToCommitLog(final long startOffset, final byte[] data); + + + /** + * 手动触发删除文件 + */ + public void excuteDeleteFilesManualy(); + + + /** + * 根据消息Key查询消息 + */ + public QueryMessageResult queryMessage(final String topic, final String key, final int maxNum, + final long begin, final long end); + + + public void updateHaMasterAddress(final String newAddr); + + + /** + * Slave落后Master多少，单位字节 + */ + public long slaveFallBehindMuch(); + + + public long now(); + + + public int cleanUnusedTopic(final Set topics); + + + /** + * 清除失效的消费队列 + */ + public void cleanExpiredConsumerQueue(); + + + /** + * 批量获取 messageId + */ + public Map getMessageIds(final String topic, int queueId, long minOffset, + final long maxOffset, SocketAddress storeHost); + + + /** + * 判断消息是否在磁盘 + */ + public boolean checkInDiskByConsumeOffset(final String topic, final int queueId, long consumeOffset); +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/PutMessageResult.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/PutMessageResult.java new file mode 100644 index 000000000..89051cf50 --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/PutMessageResult.java @@ -0,0 +1,66 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store; + +/** + * 写入消息返回结果 + * + * @author shijia.wxr + * @since 2013-7-21 + */ +public class PutMessageResult { + private PutMessageStatus putMessageStatus; + private AppendMessageResult appendMessageResult; + + + public PutMessageResult(PutMessageStatus putMessageStatus, AppendMessageResult appendMessageResult) { + this.putMessageStatus = putMessageStatus; + this.appendMessageResult = appendMessageResult; + } + + + public boolean isOk() { + return this.appendMessageResult != null && this.appendMessageResult.isOk(); + } + + + public AppendMessageResult getAppendMessageResult() { + return appendMessageResult; + } + + + public void setAppendMessageResult(AppendMessageResult appendMessageResult) { + this.appendMessageResult = appendMessageResult; + } + + + public PutMessageStatus getPutMessageStatus() { + return putMessageStatus; + } + + + public void setPutMessageStatus(PutMessageStatus putMessageStatus) { + this.putMessageStatus = putMessageStatus; + } + + + @Override + public String toString() { + return "PutMessageResult [putMessageStatus=" + putMessageStatus + ", appendMessageResult=" + + appendMessageResult + "]"; + } + +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/PutMessageStatus.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/PutMessageStatus.java new file mode 100644 index 000000000..5a307bfb1 --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/PutMessageStatus.java @@ -0,0 +1,33 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store; + +/** + * 写入消息过程的返回结果 + * + * @author shijia.wxr + * @since 2013-7-21 + */ +public enum PutMessageStatus { + PUT_OK, + FLUSH_DISK_TIMEOUT, + FLUSH_SLAVE_TIMEOUT, + SLAVE_NOT_AVAILABLE, + SERVICE_NOT_AVAILABLE, + CREATE_MAPEDFILE_FAILED, + MESSAGE_ILLEGAL, + UNKNOWN_ERROR, +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/QueryMessageResult.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/QueryMessageResult.java new file mode 100644 index 000000000..9f91fc66d --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/QueryMessageResult.java @@ -0,0 +1,83 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + + +/** + * 通过Key查询消息，返回结果 + * + * @author shijia.wxr + * @since 2013-7-21 + */ +public class QueryMessageResult { + // 多个连续的消息集合 + private final List messageMapedList = + new ArrayList(100); + // 用来向Consumer传送消息 + private final List messageBufferList = new ArrayList(100); + private long indexLastUpdateTimestamp; + private long indexLastUpdatePhyoffset; + // ByteBuffer 总字节数 + private int bufferTotalSize = 0; + + + public void addMessage(final SelectMapedBufferResult mapedBuffer) { + this.messageMapedList.add(mapedBuffer); + this.messageBufferList.add(mapedBuffer.getByteBuffer()); + this.bufferTotalSize += mapedBuffer.getSize(); + } + + + public void release() { + for (SelectMapedBufferResult select : this.messageMapedList) { + select.release(); + } + } + + + public long getIndexLastUpdateTimestamp() { + return indexLastUpdateTimestamp; + } + + + public void setIndexLastUpdateTimestamp(long indexLastUpdateTimestamp) { + this.indexLastUpdateTimestamp = indexLastUpdateTimestamp; + } + + + public long getIndexLastUpdatePhyoffset() { + return indexLastUpdatePhyoffset; + } + + + public void setIndexLastUpdatePhyoffset(long indexLastUpdatePhyoffset) { + this.indexLastUpdatePhyoffset = indexLastUpdatePhyoffset; + } + + + public List getMessageBufferList() { + return messageBufferList; + } + + + public int getBufferTotalSize() { + return bufferTotalSize; + } +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/ReferenceResource.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/ReferenceResource.java new file mode 100644 index 000000000..a6c97a8ba --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/ReferenceResource.java @@ -0,0 +1,107 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store; + +import java.util.concurrent.atomic.AtomicLong; + + +/** + * 引用计数基类，类似于C++智能指针实现 + * + * @author shijia.wxr + * @since 2013-7-21 + */ +public abstract class ReferenceResource { + protected final AtomicLong refCount = new AtomicLong(1); + protected volatile boolean available = true; + protected volatile boolean cleanupOver = false; + private volatile long firstShutdownTimestamp = 0; + + + /** + * 资源是否能HOLD住 + */ + public synchronized boolean hold() { + if (this.isAvailable()) { + if (this.refCount.getAndIncrement() > 0) { + return true; + } + else { + this.refCount.getAndDecrement(); + } + } + + return false; + } + + + /** + * 资源是否可用，即是否可被HOLD + */ + public boolean isAvailable() { + return this.available; + } + + + /** + * 禁止资源被访问 shutdown不允许调用多次，最好是由管理线程调用 + */ + public void shutdown(final long intervalForcibly) { + if (this.available) { + this.available = false; + this.firstShutdownTimestamp = System.currentTimeMillis(); + this.release(); + } + // 强制shutdown + else if (this.getRefCount() > 0) { + if ((System.currentTimeMillis() - this.firstShutdownTimestamp) >= intervalForcibly) { + this.refCount.set(-1000 - this.getRefCount()); + this.release(); + } + } + } + + + public long getRefCount() { + return this.refCount.get(); + } + + + /** + * 释放资源 + */ + public void release() { + long value = this.refCount.decrementAndGet(); + if (value > 0) + return; + + synchronized (this) { + // cleanup内部要对是否clean做处理 + this.cleanupOver = this.cleanup(value); + } + } + + + public abstract boolean cleanup(final long currentRef); + + + /** + * 资源是否被清理完成 + */ + public boolean isCleanupOver() { + return this.refCount.get() <= 0 && this.cleanupOver; + } +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/RunningFlags.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/RunningFlags.java new file mode 100644 index 000000000..89727f65b --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/RunningFlags.java @@ -0,0 +1,147 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store; + +/** + * 存储模型运行过程的状态位 + * + * @author shijia.wxr + * @since 2013-7-21 + */ +public class RunningFlags { + // 禁止读权限 + private static final int NotReadableBit = 1; + // 禁止写权限 + private static final int NotWriteableBit = 1 << 1; + // 逻辑队列是否发生错误 + private static final int WriteLogicsQueueErrorBit = 1 << 2; + // 索引文件是否发生错误 + private static final int WriteIndexFileErrorBit = 1 << 3; + // 磁盘空间不足 + private static final int DiskFullBit = 1 << 4; + private volatile int flagBits = 0; + + + public RunningFlags() { + } + + + public int getFlagBits() { + return flagBits; + } + + + public boolean getAndMakeReadable() { + boolean result = this.isReadable(); + if (!result) { + this.flagBits &= ~NotReadableBit; + } + return result; + } + + + public boolean isReadable() { + if ((this.flagBits & NotReadableBit) == 0) { + return true; + } + + return false; + } + + + public boolean getAndMakeNotReadable() { + boolean result = this.isReadable(); + if (result) { + this.flagBits |= NotReadableBit; + } + return result; + } + + + public boolean getAndMakeWriteable() { + boolean result = this.isWriteable(); + if (!result) { + this.flagBits &= ~NotWriteableBit; + } + return result; + } + + + public boolean isWriteable() { + if ((this.flagBits & (NotWriteableBit | WriteLogicsQueueErrorBit | DiskFullBit | WriteIndexFileErrorBit)) == 0) { + return true; + } + + return false; + } + + + public boolean getAndMakeNotWriteable() { + boolean result = this.isWriteable(); + if (result) { + this.flagBits |= NotWriteableBit; + } + return result; + } + + + public void makeLogicsQueueError() { + this.flagBits |= WriteLogicsQueueErrorBit; + } + + + public boolean isLogicsQueueError() { + if ((this.flagBits & WriteLogicsQueueErrorBit) == WriteLogicsQueueErrorBit) { + return true; + } + + return false; + } + + + public void makeIndexFileError() { + this.flagBits |= WriteIndexFileErrorBit; + } + + + public boolean isIndexFileError() { + if ((this.flagBits & WriteIndexFileErrorBit) == WriteIndexFileErrorBit) { + return true; + } + + return false; + } + + + /** + * 返回Disk是否正常 + */ + public boolean getAndMakeDiskFull() { + boolean result = !((this.flagBits & DiskFullBit) == DiskFullBit); + this.flagBits |= DiskFullBit; + return result; + } + + + /** + * 返回Disk是否正常 + */ + public boolean getAndMakeDiskOK() { + boolean result = !((this.flagBits & DiskFullBit) == DiskFullBit); + this.flagBits &= ~DiskFullBit; + return result; + } +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/SelectMapedBufferResult.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/SelectMapedBufferResult.java new file mode 100644 index 000000000..4c71a3711 --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/SelectMapedBufferResult.java @@ -0,0 +1,89 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store; + +import java.nio.ByteBuffer; + + +/** + * 查询Pagecache返回结果 + * + * @author shijia.wxr + * @since 2013-7-21 + */ +public class SelectMapedBufferResult { + // 从队列中哪个绝对Offset开始 + private final long startOffset; + // position从0开始 + private final ByteBuffer byteBuffer; + // 有效数据大小 + private int size; + // 用来释放内存 + private MapedFile mapedFile; + + + public SelectMapedBufferResult(long startOffset, ByteBuffer byteBuffer, int size, MapedFile mapedFile) { + this.startOffset = startOffset; + this.byteBuffer = byteBuffer; + this.size = size; + this.mapedFile = mapedFile; + } + + + public ByteBuffer getByteBuffer() { + return byteBuffer; + } + + + public int getSize() { + return size; + } + + + public void setSize(final int s) { + this.size = s; + this.byteBuffer.limit(this.size); + } + + + public MapedFile getMapedFile() { + return mapedFile; + } + + + @Override + protected void finalize() { + if (this.mapedFile != null) { + this.release(); + } + } + + + /** + * 此方法只能被调用一次，重复调用无效 + */ + public synchronized void release() { + if (this.mapedFile != null) { + this.mapedFile.release(); + this.mapedFile = null; + } + } + + + public long getStartOffset() { + return startOffset; + } +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/StoreCheckpoint.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/StoreCheckpoint.java new file mode 100644 index 000000000..6b1f39473 --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/StoreCheckpoint.java @@ -0,0 +1,146 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store; + +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.MappedByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.channels.FileChannel.MapMode; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.constant.LoggerName; + + +/** + * 记录存储模型最终一致的时间点 + * + * @author shijia.wxr + * @since 2013-7-21 + */ +public class StoreCheckpoint { + private static final Logger log = LoggerFactory.getLogger(LoggerName.StoreLoggerName); + private final RandomAccessFile randomAccessFile; + private final FileChannel fileChannel; + private final MappedByteBuffer mappedByteBuffer; + private volatile long physicMsgTimestamp = 0; + private volatile long logicsMsgTimestamp = 0; + private volatile long indexMsgTimestamp = 0; + + + public StoreCheckpoint(final String scpPath) throws IOException { + File file = new File(scpPath); + MapedFile.ensureDirOK(file.getParent()); + boolean fileExists = file.exists(); + + this.randomAccessFile = new RandomAccessFile(file, "rw"); + this.fileChannel = this.randomAccessFile.getChannel(); + this.mappedByteBuffer = fileChannel.map(MapMode.READ_WRITE, 0, MapedFile.OS_PAGE_SIZE); + + if (fileExists) { + log.info("store checkpoint file exists, " + scpPath); + this.physicMsgTimestamp = this.mappedByteBuffer.getLong(0); + this.logicsMsgTimestamp = this.mappedByteBuffer.getLong(8); + this.indexMsgTimestamp = this.mappedByteBuffer.getLong(16); + + log.info("store checkpoint file physicMsgTimestamp " + this.physicMsgTimestamp + ", " + + UtilAll.timeMillisToHumanString(this.physicMsgTimestamp)); + log.info("store checkpoint file logicsMsgTimestamp " + this.logicsMsgTimestamp + ", " + + UtilAll.timeMillisToHumanString(this.logicsMsgTimestamp)); + log.info("store checkpoint file indexMsgTimestamp " + this.indexMsgTimestamp + ", " + + UtilAll.timeMillisToHumanString(this.indexMsgTimestamp)); + } + else { + log.info("store checkpoint file not exists, " + scpPath); + } + } + + + public void shutdown() { + this.flush(); + + // unmap mappedByteBuffer + MapedFile.clean(this.mappedByteBuffer); + + try { + this.fileChannel.close(); + } + catch (IOException e) { + e.printStackTrace(); + } + } + + + public void flush() { + this.mappedByteBuffer.putLong(0, this.physicMsgTimestamp); + this.mappedByteBuffer.putLong(8, this.logicsMsgTimestamp); + this.mappedByteBuffer.putLong(16, this.indexMsgTimestamp); + this.mappedByteBuffer.force(); + } + + + public long getPhysicMsgTimestamp() { + return physicMsgTimestamp; + } + + + public void setPhysicMsgTimestamp(long physicMsgTimestamp) { + this.physicMsgTimestamp = physicMsgTimestamp; + } + + + public long getLogicsMsgTimestamp() { + return logicsMsgTimestamp; + } + + + public void setLogicsMsgTimestamp(long logicsMsgTimestamp) { + this.logicsMsgTimestamp = logicsMsgTimestamp; + } + + + public long getMinTimestampIndex() { + return Math.min(this.getMinTimestamp(), this.indexMsgTimestamp); + } + + + public long getMinTimestamp() { + long min = Math.min(this.physicMsgTimestamp, this.logicsMsgTimestamp); + + // 向前倒退3s，防止因为时间精度问题导致丢数据 + // fixed https://github.com/alibaba/RocketMQ/issues/467 + min -= 1000 * 3; + if (min < 0) + min = 0; + + return min; + } + + + public long getIndexMsgTimestamp() { + return indexMsgTimestamp; + } + + + public void setIndexMsgTimestamp(long indexMsgTimestamp) { + this.indexMsgTimestamp = indexMsgTimestamp; + } + +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/StoreStatsService.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/StoreStatsService.java new file mode 100644 index 000000000..5e5677709 --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/StoreStatsService.java @@ -0,0 +1,610 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store; + +import java.text.MessageFormat; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.ReentrantLock; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.rocketmq.common.ServiceThread; +import com.alibaba.rocketmq.common.constant.LoggerName; + + +/** + * 存储层内部统计服务 + * + * @author shijia.wxr + * @since 2013-7-21 + */ +public class StoreStatsService extends ServiceThread { + private static final Logger log = LoggerFactory.getLogger(LoggerName.StoreLoggerName); + // 采样频率，1秒钟采样一次 + private static final int FrequencyOfSampling = 1000; + // 采样最大记录数，超过则将之前的删除掉 + private static final int MaxRecordsOfSampling = 60 * 10; + // 打印TPS数据间隔时间，单位秒，1分钟 + private static int PrintTPSInterval = 60 * 1; + // putMessage，失败次数 + private final AtomicLong putMessageFailedTimes = new AtomicLong(0); + // putMessage，调用总数 + private final Map putMessageTopicTimesTotal = + new ConcurrentHashMap(128); + // putMessage，Message Size Total + private final Map putMessageTopicSizeTotal = + new ConcurrentHashMap(128); + // getMessage，调用总数 + private final AtomicLong getMessageTimesTotalFound = new AtomicLong(0); + private final AtomicLong getMessageTransferedMsgCount = new AtomicLong(0); + private final AtomicLong getMessageTimesTotalMiss = new AtomicLong(0); + // putMessage，耗时分布 + private final AtomicLong[] putMessageDistributeTime = new AtomicLong[7]; + // put最近10分钟采样 + private final LinkedList putTimesList = new LinkedList(); + // get最近10分钟采样 + private final LinkedList getTimesFoundList = new LinkedList(); + private final LinkedList getTimesMissList = new LinkedList(); + private final LinkedList transferedMsgCountList = new LinkedList(); + // 启动时间 + private long messageStoreBootTimestamp = System.currentTimeMillis(); + // putMessage，写入整个消息耗时，含加锁竟争时间（单位毫秒） + private volatile long putMessageEntireTimeMax = 0; + // getMessage，读取一批消息耗时，含加锁竟争时间（单位毫秒） + private volatile long getMessageEntireTimeMax = 0; + // for putMessageEntireTimeMax + private ReentrantLock lockPut = new ReentrantLock(); + // for getMessageEntireTimeMax + private ReentrantLock lockGet = new ReentrantLock(); + // DispatchMessageService，缓冲区最大值 + private volatile long dispatchMaxBuffer = 0; + // 针对采样线程加锁 + private ReentrantLock lockSampling = new ReentrantLock(); + private long lastPrintTimestamp = System.currentTimeMillis(); + + + public StoreStatsService() { + for (int i = 0; i < this.putMessageDistributeTime.length; i++) { + putMessageDistributeTime[i] = new AtomicLong(0); + } + } + + + public long getPutMessageEntireTimeMax() { + return putMessageEntireTimeMax; + } + + + public void setPutMessageEntireTimeMax(long value) { + // 微秒 + if (value <= 0) { + this.putMessageDistributeTime[0].incrementAndGet(); + } + // 几毫秒 + else if (value < 10) { + this.putMessageDistributeTime[1].incrementAndGet(); + } + // 几十毫秒 + else if (value < 100) { + this.putMessageDistributeTime[2].incrementAndGet(); + } + // 几百毫秒（500毫秒以内） + else if (value < 500) { + this.putMessageDistributeTime[3].incrementAndGet(); + } + // 几百毫秒（500毫秒以上） + else if (value < 1000) { + this.putMessageDistributeTime[4].incrementAndGet(); + } + // 几秒 + else if (value < 10000) { + this.putMessageDistributeTime[5].incrementAndGet(); + } + // 大等于10秒 + else { + this.putMessageDistributeTime[6].incrementAndGet(); + } + + if (value > this.putMessageEntireTimeMax) { + this.lockPut.lock(); + this.putMessageEntireTimeMax = + value > this.putMessageEntireTimeMax ? value : this.putMessageEntireTimeMax; + this.lockPut.unlock(); + } + } + + + public long getGetMessageEntireTimeMax() { + return getMessageEntireTimeMax; + } + + + public void setGetMessageEntireTimeMax(long value) { + if (value > this.getMessageEntireTimeMax) { + this.lockGet.lock(); + this.getMessageEntireTimeMax = + value > this.getMessageEntireTimeMax ? value : this.getMessageEntireTimeMax; + this.lockGet.unlock(); + } + } + + + public long getDispatchMaxBuffer() { + return dispatchMaxBuffer; + } + + + public void setDispatchMaxBuffer(long value) { + this.dispatchMaxBuffer = value > this.dispatchMaxBuffer ? value : this.dispatchMaxBuffer; + } + + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder(1024); + Long totalTimes = getPutMessageTimesTotal(); + if (0 == totalTimes) { + totalTimes = 1L; + } + + sb.append("\truntime: " + this.getFormatRuntime() + "\r\n"); + sb.append("\tputMessageEntireTimeMax: " + this.putMessageEntireTimeMax + "\r\n"); + sb.append("\tputMessageTimesTotal: " + totalTimes + "\r\n"); + sb.append("\tputMessageSizeTotal: " + this.getPutMessageSizeTotal() + "\r\n"); + sb.append("\tputMessageDistributeTime: " + this.getPutMessageDistributeTimeStringInfo(totalTimes) + + "\r\n"); + sb.append("\tputMessageAverageSize: " + (this.getPutMessageSizeTotal() / totalTimes.doubleValue()) + + "\r\n"); + sb.append("\tdispatchMaxBuffer: " + this.dispatchMaxBuffer + "\r\n"); + sb.append("\tgetMessageEntireTimeMax: " + this.getMessageEntireTimeMax + "\r\n"); + sb.append("\tputTps: " + this.getPutTps() + "\r\n"); + sb.append("\tgetFoundTps: " + this.getGetFoundTps() + "\r\n"); + sb.append("\tgetMissTps: " + this.getGetMissTps() + "\r\n"); + sb.append("\tgetTotalTps: " + this.getGetTotalTps() + "\r\n"); + sb.append("\tgetTransferedTps: " + this.getGetTransferedTps() + "\r\n"); + return sb.toString(); + } + + + private String getPutMessageDistributeTimeStringInfo(Long total) { + final StringBuilder sb = new StringBuilder(512); + + for (AtomicLong i : this.putMessageDistributeTime) { + long value = i.get(); + double ratio = value / total.doubleValue(); + sb.append("\r\n\t\t"); + sb.append(value + "(" + (ratio * 100) + "%)"); + } + + return sb.toString(); + } + + + private String getFormatRuntime() { + final long MILLISECOND = 1; + final long SECOND = 1000 * MILLISECOND; + final long MINUTE = 60 * SECOND; + final long HOUR = 60 * MINUTE; + final long DAY = 24 * HOUR; + final MessageFormat TIME = new MessageFormat("[ {0} days, {1} hours, {2} minutes, {3} seconds ]"); + + long time = System.currentTimeMillis() - this.messageStoreBootTimestamp; + long days = time / DAY; + long hours = (time % DAY) / HOUR; + long minutes = (time % HOUR) / MINUTE; + long seconds = (time % MINUTE) / SECOND; + return TIME.format(new Long[] { days, hours, minutes, seconds }); + } + + + private String getPutTps() { + StringBuilder sb = new StringBuilder(); + // 10秒钟 + sb.append(this.getPutTps(10)); + sb.append(" "); + + // 1分钟 + sb.append(this.getPutTps(60)); + sb.append(" "); + + // 10分钟 + sb.append(this.getPutTps(600)); + + return sb.toString(); + } + + + private String getPutTps(int time) { + String result = ""; + this.lockSampling.lock(); + try { + CallSnapshot last = this.putTimesList.getLast(); + + if (this.putTimesList.size() > time) { + CallSnapshot lastBefore = this.putTimesList.get(this.putTimesList.size() - (time + 1)); + result += CallSnapshot.getTPS(lastBefore, last); + } + + } + finally { + this.lockSampling.unlock(); + } + return result; + } + + + private String getGetFoundTps() { + StringBuilder sb = new StringBuilder(); + // 10秒钟 + sb.append(this.getGetFoundTps(10)); + sb.append(" "); + + // 1分钟 + sb.append(this.getGetFoundTps(60)); + sb.append(" "); + + // 10分钟 + sb.append(this.getGetFoundTps(600)); + + return sb.toString(); + } + + + private String getGetFoundTps(int time) { + String result = ""; + this.lockSampling.lock(); + try { + CallSnapshot last = this.getTimesFoundList.getLast(); + + if (this.getTimesFoundList.size() > time) { + CallSnapshot lastBefore = + this.getTimesFoundList.get(this.getTimesFoundList.size() - (time + 1)); + result += CallSnapshot.getTPS(lastBefore, last); + } + } + finally { + this.lockSampling.unlock(); + } + + return result; + } + + + private String getGetMissTps() { + StringBuilder sb = new StringBuilder(); + // 10秒钟 + sb.append(this.getGetMissTps(10)); + sb.append(" "); + + // 1分钟 + sb.append(this.getGetMissTps(60)); + sb.append(" "); + + // 10分钟 + sb.append(this.getGetMissTps(600)); + + return sb.toString(); + } + + + private String getGetMissTps(int time) { + String result = ""; + this.lockSampling.lock(); + try { + CallSnapshot last = this.getTimesMissList.getLast(); + + if (this.getTimesMissList.size() > time) { + CallSnapshot lastBefore = + this.getTimesMissList.get(this.getTimesMissList.size() - (time + 1)); + result += CallSnapshot.getTPS(lastBefore, last); + } + + } + finally { + this.lockSampling.unlock(); + } + + return result; + } + + + private String getGetTransferedTps() { + StringBuilder sb = new StringBuilder(); + // 10秒钟 + sb.append(this.getGetTransferedTps(10)); + sb.append(" "); + + // 1分钟 + sb.append(this.getGetTransferedTps(60)); + sb.append(" "); + + // 10分钟 + sb.append(this.getGetTransferedTps(600)); + + return sb.toString(); + } + + + private String getGetTransferedTps(int time) { + String result = ""; + this.lockSampling.lock(); + try { + CallSnapshot last = this.transferedMsgCountList.getLast(); + + if (this.transferedMsgCountList.size() > time) { + CallSnapshot lastBefore = + this.transferedMsgCountList.get(this.transferedMsgCountList.size() - (time + 1)); + result += CallSnapshot.getTPS(lastBefore, last); + } + + } + finally { + this.lockSampling.unlock(); + } + + return result; + } + + + private String getGetTotalTps() { + StringBuilder sb = new StringBuilder(); + // 10秒钟 + sb.append(this.getGetTotalTps(10)); + sb.append(" "); + + // 1分钟 + sb.append(this.getGetTotalTps(60)); + sb.append(" "); + + // 10分钟 + sb.append(this.getGetTotalTps(600)); + + return sb.toString(); + } + + + private String getGetTotalTps(int time) { + this.lockSampling.lock(); + double found = 0; + double miss = 0; + try { + { + CallSnapshot last = this.getTimesFoundList.getLast(); + + if (this.getTimesFoundList.size() > time) { + CallSnapshot lastBefore = + this.getTimesFoundList.get(this.getTimesFoundList.size() - (time + 1)); + found = CallSnapshot.getTPS(lastBefore, last); + } + } + { + CallSnapshot last = this.getTimesMissList.getLast(); + + if (this.getTimesMissList.size() > time) { + CallSnapshot lastBefore = + this.getTimesMissList.get(this.getTimesMissList.size() - (time + 1)); + miss = CallSnapshot.getTPS(lastBefore, last); + } + } + + } + finally { + this.lockSampling.unlock(); + } + + return Double.toString(found + miss); + } + + + public long getPutMessageTimesTotal() { + long rs = 0; + for (AtomicLong data : putMessageTopicTimesTotal.values()) { + rs += data.get(); + } + return rs; + } + + + public long getPutMessageSizeTotal() { + long rs = 0; + for (AtomicLong data : putMessageTopicSizeTotal.values()) { + rs += data.get(); + } + return rs; + } + + + public HashMap getRuntimeInfo() { + HashMap result = new HashMap(64); + + Long totalTimes = getPutMessageTimesTotal(); + if (0 == totalTimes) { + totalTimes = 1L; + } + + result.put("bootTimestamp", String.valueOf(this.messageStoreBootTimestamp)); + result.put("runtime", this.getFormatRuntime()); + result.put("putMessageEntireTimeMax", String.valueOf(this.putMessageEntireTimeMax)); + result.put("putMessageTimesTotal", String.valueOf(totalTimes)); + result.put("putMessageSizeTotal", String.valueOf(this.getPutMessageSizeTotal())); + result.put("putMessageDistributeTime", + String.valueOf(this.getPutMessageDistributeTimeStringInfo(totalTimes))); + result.put("putMessageAverageSize", + String.valueOf((this.getPutMessageSizeTotal() / totalTimes.doubleValue()))); + result.put("dispatchMaxBuffer", String.valueOf(this.dispatchMaxBuffer)); + result.put("getMessageEntireTimeMax", String.valueOf(this.getMessageEntireTimeMax)); + result.put("putTps", String.valueOf(this.getPutTps())); + result.put("getFoundTps", String.valueOf(this.getGetFoundTps())); + result.put("getMissTps", String.valueOf(this.getGetMissTps())); + result.put("getTotalTps", String.valueOf(this.getGetTotalTps())); + result.put("getTransferedTps", String.valueOf(this.getGetTransferedTps())); + + return result; + } + + + public void run() { + log.info(this.getServiceName() + " service started"); + + while (!this.isStoped()) { + try { + this.waitForRunning(FrequencyOfSampling); + + this.sampling(); + + this.printTps(); + } + catch (Exception e) { + log.warn(this.getServiceName() + " service has exception. ", e); + } + } + + log.info(this.getServiceName() + " service end"); + } + + + private void sampling() { + this.lockSampling.lock(); + try { + this.putTimesList.add(new CallSnapshot(System.currentTimeMillis(), getPutMessageTimesTotal())); + if (this.putTimesList.size() > (MaxRecordsOfSampling + 1)) { + this.putTimesList.removeFirst(); + } + + this.getTimesFoundList.add(new CallSnapshot(System.currentTimeMillis(), + this.getMessageTimesTotalFound.get())); + if (this.getTimesFoundList.size() > (MaxRecordsOfSampling + 1)) { + this.getTimesFoundList.removeFirst(); + } + + this.getTimesMissList.add(new CallSnapshot(System.currentTimeMillis(), + this.getMessageTimesTotalMiss.get())); + if (this.getTimesMissList.size() > (MaxRecordsOfSampling + 1)) { + this.getTimesMissList.removeFirst(); + } + + this.transferedMsgCountList.add(new CallSnapshot(System.currentTimeMillis(), + this.getMessageTransferedMsgCount.get())); + if (this.transferedMsgCountList.size() > (MaxRecordsOfSampling + 1)) { + this.transferedMsgCountList.removeFirst(); + } + + } + finally { + this.lockSampling.unlock(); + } + } + + + /** + * 1分钟打印一次TPS + */ + private void printTps() { + if (System.currentTimeMillis() > (this.lastPrintTimestamp + PrintTPSInterval * 1000)) { + this.lastPrintTimestamp = System.currentTimeMillis(); + + log.info("put_tps {}", this.getPutTps(PrintTPSInterval)); + + log.info("get_found_tps {}", this.getGetFoundTps(PrintTPSInterval)); + + log.info("get_miss_tps {}", this.getGetMissTps(PrintTPSInterval)); + + log.info("get_transfered_tps {}", this.getGetTransferedTps(PrintTPSInterval)); + } + } + + + @Override + public String getServiceName() { + return StoreStatsService.class.getSimpleName(); + } + + + public AtomicLong getGetMessageTimesTotalFound() { + return getMessageTimesTotalFound; + } + + + public AtomicLong getGetMessageTimesTotalMiss() { + return getMessageTimesTotalMiss; + } + + + public AtomicLong getGetMessageTransferedMsgCount() { + return getMessageTransferedMsgCount; + } + + + public AtomicLong getPutMessageFailedTimes() { + return putMessageFailedTimes; + } + + + public AtomicLong getSinglePutMessageTopicSizeTotal(String topic) { + AtomicLong rs = putMessageTopicSizeTotal.get(topic); + if (null == rs) { + rs = new AtomicLong(0); + putMessageTopicSizeTotal.put(topic, rs); + } + return rs; + } + + + public AtomicLong getSinglePutMessageTopicTimesTotal(String topic) { + AtomicLong rs = putMessageTopicTimesTotal.get(topic); + if (null == rs) { + rs = new AtomicLong(0); + putMessageTopicTimesTotal.put(topic, rs); + } + return rs; + } + + + public Map getPutMessageTopicTimesTotal() { + return putMessageTopicTimesTotal; + } + + + public Map getPutMessageTopicSizeTotal() { + return putMessageTopicSizeTotal; + } + + static class CallSnapshot { + public final long timestamp; + public final long callTimesTotal; + + + public CallSnapshot(long timestamp, long callTimesTotal) { + this.timestamp = timestamp; + this.callTimesTotal = callTimesTotal; + } + + + public static double getTPS(final CallSnapshot begin, final CallSnapshot end) { + long total = end.callTimesTotal - begin.callTimesTotal; + Long time = end.timestamp - begin.timestamp; + + double tps = total / time.doubleValue(); + + return tps * 1000; + } + } +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/StoreUtil.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/StoreUtil.java new file mode 100644 index 000000000..9509882bc --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/StoreUtil.java @@ -0,0 +1,40 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store; + +import java.lang.management.ManagementFactory; +import java.lang.management.OperatingSystemMXBean; + + +/** + * @author shijia.wxr + * @since 13-8-3 + */ +public class StoreUtil { + public static final long TotalPhysicalMemorySize = getTotalPhysicalMemorySize(); + + + @SuppressWarnings("restriction") + public static long getTotalPhysicalMemorySize() { + long physicalTotal = 1024 * 1024 * 1024 * 24; + OperatingSystemMXBean osmxb = ManagementFactory.getOperatingSystemMXBean(); + if (osmxb instanceof com.sun.management.OperatingSystemMXBean) { + physicalTotal = ((com.sun.management.OperatingSystemMXBean) osmxb).getTotalPhysicalMemorySize(); + } + + return physicalTotal; + } +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/config/BrokerRole.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/config/BrokerRole.java new file mode 100644 index 000000000..51f4afc71 --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/config/BrokerRole.java @@ -0,0 +1,31 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store.config; + +/** + * Broker角色 + * + * @author shijia.wxr + * @since 2013-7-21 + */ +public enum BrokerRole { + // 异步复制Master + ASYNC_MASTER, + // 同步双写Master + SYNC_MASTER, + // Slave + SLAVE +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/config/FlushDiskType.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/config/FlushDiskType.java new file mode 100644 index 000000000..5773a25ea --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/config/FlushDiskType.java @@ -0,0 +1,33 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store.config; + +/** + * 刷盘方式 + * + * @author shijia.wxr + * @since 2013-7-21 + */ +public enum FlushDiskType { + /** + * 同步刷盘 + */ + SYNC_FLUSH, + /** + * 异步刷盘 + */ + ASYNC_FLUSH +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/config/MessageStoreConfig.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/config/MessageStoreConfig.java new file mode 100644 index 000000000..75121967e --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/config/MessageStoreConfig.java @@ -0,0 +1,595 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store.config; + +import java.io.File; + +import com.alibaba.rocketmq.common.annotation.ImportantField; +import com.alibaba.rocketmq.store.ConsumeQueue; + + +/** + * 存储层配置文件类 + * + * @author shijia.wxr + * @since 2013-7-21 + */ +public class MessageStoreConfig { + // 存储跟目录 + @ImportantField + private String storePathRootDir = System.getProperty("user.home") + File.separator + "store"; + + // CommitLog存储目录 + @ImportantField + private String storePathCommitLog = System.getProperty("user.home") + File.separator + "store" + + File.separator + "commitlog"; + + // CommitLog每个文件大小 1G + private int mapedFileSizeCommitLog = 1024 * 1024 * 1024; + // ConsumeQueue每个文件大小 默认存储30W条消息 + private int mapedFileSizeConsumeQueue = 300000 * ConsumeQueue.CQStoreUnitSize; + // CommitLog刷盘间隔时间（单位毫秒） + @ImportantField + private int flushIntervalCommitLog = 1000; + // 是否定时方式刷盘，默认是实时刷盘 + @ImportantField + private boolean flushCommitLogTimed = false; + // ConsumeQueue刷盘间隔时间（单位毫秒） + private int flushIntervalConsumeQueue = 1000; + // 清理资源间隔时间（单位毫秒） + private int cleanResourceInterval = 10000; + // 删除多个CommitLog文件的间隔时间（单位毫秒） + private int deleteCommitLogFilesInterval = 100; + // 删除多个ConsumeQueue文件的间隔时间（单位毫秒） + private int deleteConsumeQueueFilesInterval = 100; + // 强制删除文件间隔时间（单位毫秒） + private int destroyMapedFileIntervalForcibly = 1000 * 120; + // 定期检查Hanged文件间隔时间（单位毫秒） + private int redeleteHangedFileInterval = 1000 * 120; + // 何时触发删除文件, 默认凌晨4点删除文件 + @ImportantField + private String deleteWhen = "04"; + // 磁盘空间最大使用率 + private int diskMaxUsedSpaceRatio = 75; + // 文件保留时间（单位小时） + @ImportantField + private int fileReservedTime = 72; + // 写消息索引到ConsumeQueue，缓冲区高水位，超过则开始流控 + private int putMsgIndexHightWater = 600000; + // 最大消息大小，默认512K + private int maxMessageSize = 1024 * 512; + // 重启时，是否校验CRC + private boolean checkCRCOnRecover = true; + // 刷CommitLog，至少刷几个PAGE + private int flushCommitLogLeastPages = 4; + // 刷ConsumeQueue，至少刷几个PAGE + private int flushConsumeQueueLeastPages = 2; + // 刷CommitLog，彻底刷盘间隔时间 + private int flushCommitLogThoroughInterval = 1000 * 10; + // 刷ConsumeQueue，彻底刷盘间隔时间 + private int flushConsumeQueueThoroughInterval = 1000 * 60; + // 最大被拉取的消息字节数，消息在内存 + @ImportantField + private int maxTransferBytesOnMessageInMemory = 1024 * 256; + // 最大被拉取的消息个数，消息在内存 + @ImportantField + private int maxTransferCountOnMessageInMemory = 32; + // 最大被拉取的消息字节数，消息在磁盘 + @ImportantField + private int maxTransferBytesOnMessageInDisk = 1024 * 64; + // 最大被拉取的消息个数，消息在磁盘 + @ImportantField + private int maxTransferCountOnMessageInDisk = 8; + // 命中消息在内存的最大比例 + @ImportantField + private int accessMessageInMemoryMaxRatio = 40; + // 是否开启消息索引功能 + @ImportantField + private boolean messageIndexEnable = true; + private int maxHashSlotNum = 5000000; + private int maxIndexNum = 5000000 * 4; + private int maxMsgsNumBatch = 64; + // 是否使用安全的消息索引功能，即可靠模式。 + // 可靠模式下，异常宕机恢复慢 + // 非可靠模式下，异常宕机恢复快 + @ImportantField + private boolean messageIndexSafe = false; + // HA功能 + private int haListenPort = 10912; + private int haSendHeartbeatInterval = 1000 * 5; + private int haHousekeepingInterval = 1000 * 20; + private int haTransferBatchSize = 1024 * 32; + // 如果不设置，则从NameServer获取Master HA服务地址 + @ImportantField + private String haMasterAddress = null; + // Slave落后Master超过此值，则认为存在异常 + private int haSlaveFallbehindMax = 1024 * 1024 * 256; + @ImportantField + private BrokerRole brokerRole = BrokerRole.ASYNC_MASTER; + @ImportantField + private FlushDiskType flushDiskType = FlushDiskType.ASYNC_FLUSH; + // 同步刷盘超时时间 + private int syncFlushTimeout = 1000 * 5; + // 定时消息相关 + private String messageDelayLevel = "1s 5s 10s 30s 1m 2m 3m 4m 5m 6m 7m 8m 9m 10m 20m 30m 1h 2h"; + private long flushDelayOffsetInterval = 1000 * 10; + // 磁盘空间超过90%警戒水位，自动开始删除文件 + @ImportantField + private boolean cleanFileForciblyEnable = true; + + + public int getMapedFileSizeCommitLog() { + return mapedFileSizeCommitLog; + } + + + public void setMapedFileSizeCommitLog(int mapedFileSizeCommitLog) { + this.mapedFileSizeCommitLog = mapedFileSizeCommitLog; + } + + + public int getMapedFileSizeConsumeQueue() { + // 此处需要向上取整 + int factor = (int) Math.ceil(this.mapedFileSizeConsumeQueue / (ConsumeQueue.CQStoreUnitSize * 1.0)); + return (int) (factor * ConsumeQueue.CQStoreUnitSize); + } + + + public void setMapedFileSizeConsumeQueue(int mapedFileSizeConsumeQueue) { + this.mapedFileSizeConsumeQueue = mapedFileSizeConsumeQueue; + } + + + public int getFlushIntervalCommitLog() { + return flushIntervalCommitLog; + } + + + public void setFlushIntervalCommitLog(int flushIntervalCommitLog) { + this.flushIntervalCommitLog = flushIntervalCommitLog; + } + + + public int getFlushIntervalConsumeQueue() { + return flushIntervalConsumeQueue; + } + + + public void setFlushIntervalConsumeQueue(int flushIntervalConsumeQueue) { + this.flushIntervalConsumeQueue = flushIntervalConsumeQueue; + } + + + public int getPutMsgIndexHightWater() { + return putMsgIndexHightWater; + } + + + public void setPutMsgIndexHightWater(int putMsgIndexHightWater) { + this.putMsgIndexHightWater = putMsgIndexHightWater; + } + + + public int getCleanResourceInterval() { + return cleanResourceInterval; + } + + + public void setCleanResourceInterval(int cleanResourceInterval) { + this.cleanResourceInterval = cleanResourceInterval; + } + + + public int getMaxMessageSize() { + return maxMessageSize; + } + + + public void setMaxMessageSize(int maxMessageSize) { + this.maxMessageSize = maxMessageSize; + } + + + public boolean isCheckCRCOnRecover() { + return checkCRCOnRecover; + } + + + public boolean getCheckCRCOnRecover() { + return checkCRCOnRecover; + } + + + public void setCheckCRCOnRecover(boolean checkCRCOnRecover) { + this.checkCRCOnRecover = checkCRCOnRecover; + } + + + public String getStorePathCommitLog() { + return storePathCommitLog; + } + + + public void setStorePathCommitLog(String storePathCommitLog) { + this.storePathCommitLog = storePathCommitLog; + } + + + public String getDeleteWhen() { + return deleteWhen; + } + + + public void setDeleteWhen(String deleteWhen) { + this.deleteWhen = deleteWhen; + } + + + public int getDiskMaxUsedSpaceRatio() { + if (this.diskMaxUsedSpaceRatio < 10) + return 10; + + if (this.diskMaxUsedSpaceRatio > 95) + return 95; + + return diskMaxUsedSpaceRatio; + } + + + public void setDiskMaxUsedSpaceRatio(int diskMaxUsedSpaceRatio) { + this.diskMaxUsedSpaceRatio = diskMaxUsedSpaceRatio; + } + + + public int getDeleteCommitLogFilesInterval() { + return deleteCommitLogFilesInterval; + } + + + public void setDeleteCommitLogFilesInterval(int deleteCommitLogFilesInterval) { + this.deleteCommitLogFilesInterval = deleteCommitLogFilesInterval; + } + + + public int getDeleteConsumeQueueFilesInterval() { + return deleteConsumeQueueFilesInterval; + } + + + public void setDeleteConsumeQueueFilesInterval(int deleteConsumeQueueFilesInterval) { + this.deleteConsumeQueueFilesInterval = deleteConsumeQueueFilesInterval; + } + + + public int getMaxTransferBytesOnMessageInMemory() { + return maxTransferBytesOnMessageInMemory; + } + + + public void setMaxTransferBytesOnMessageInMemory(int maxTransferBytesOnMessageInMemory) { + this.maxTransferBytesOnMessageInMemory = maxTransferBytesOnMessageInMemory; + } + + + public int getMaxTransferCountOnMessageInMemory() { + return maxTransferCountOnMessageInMemory; + } + + + public void setMaxTransferCountOnMessageInMemory(int maxTransferCountOnMessageInMemory) { + this.maxTransferCountOnMessageInMemory = maxTransferCountOnMessageInMemory; + } + + + public int getMaxTransferBytesOnMessageInDisk() { + return maxTransferBytesOnMessageInDisk; + } + + + public void setMaxTransferBytesOnMessageInDisk(int maxTransferBytesOnMessageInDisk) { + this.maxTransferBytesOnMessageInDisk = maxTransferBytesOnMessageInDisk; + } + + + public int getMaxTransferCountOnMessageInDisk() { + return maxTransferCountOnMessageInDisk; + } + + + public void setMaxTransferCountOnMessageInDisk(int maxTransferCountOnMessageInDisk) { + this.maxTransferCountOnMessageInDisk = maxTransferCountOnMessageInDisk; + } + + + public int getFlushCommitLogLeastPages() { + return flushCommitLogLeastPages; + } + + + public void setFlushCommitLogLeastPages(int flushCommitLogLeastPages) { + this.flushCommitLogLeastPages = flushCommitLogLeastPages; + } + + + public int getFlushConsumeQueueLeastPages() { + return flushConsumeQueueLeastPages; + } + + + public void setFlushConsumeQueueLeastPages(int flushConsumeQueueLeastPages) { + this.flushConsumeQueueLeastPages = flushConsumeQueueLeastPages; + } + + + public int getFlushCommitLogThoroughInterval() { + return flushCommitLogThoroughInterval; + } + + + public void setFlushCommitLogThoroughInterval(int flushCommitLogThoroughInterval) { + this.flushCommitLogThoroughInterval = flushCommitLogThoroughInterval; + } + + + public int getFlushConsumeQueueThoroughInterval() { + return flushConsumeQueueThoroughInterval; + } + + + public void setFlushConsumeQueueThoroughInterval(int flushConsumeQueueThoroughInterval) { + this.flushConsumeQueueThoroughInterval = flushConsumeQueueThoroughInterval; + } + + + public int getDestroyMapedFileIntervalForcibly() { + return destroyMapedFileIntervalForcibly; + } + + + public void setDestroyMapedFileIntervalForcibly(int destroyMapedFileIntervalForcibly) { + this.destroyMapedFileIntervalForcibly = destroyMapedFileIntervalForcibly; + } + + + public int getFileReservedTime() { + return fileReservedTime; + } + + + public void setFileReservedTime(int fileReservedTime) { + this.fileReservedTime = fileReservedTime; + } + + + public int getRedeleteHangedFileInterval() { + return redeleteHangedFileInterval; + } + + + public void setRedeleteHangedFileInterval(int redeleteHangedFileInterval) { + this.redeleteHangedFileInterval = redeleteHangedFileInterval; + } + + + public int getAccessMessageInMemoryMaxRatio() { + return accessMessageInMemoryMaxRatio; + } + + + public void setAccessMessageInMemoryMaxRatio(int accessMessageInMemoryMaxRatio) { + this.accessMessageInMemoryMaxRatio = accessMessageInMemoryMaxRatio; + } + + + public boolean isMessageIndexEnable() { + return messageIndexEnable; + } + + + public void setMessageIndexEnable(boolean messageIndexEnable) { + this.messageIndexEnable = messageIndexEnable; + } + + + public int getMaxHashSlotNum() { + return maxHashSlotNum; + } + + + public void setMaxHashSlotNum(int maxHashSlotNum) { + this.maxHashSlotNum = maxHashSlotNum; + } + + + public int getMaxIndexNum() { + return maxIndexNum; + } + + + public void setMaxIndexNum(int maxIndexNum) { + this.maxIndexNum = maxIndexNum; + } + + + public int getMaxMsgsNumBatch() { + return maxMsgsNumBatch; + } + + + public void setMaxMsgsNumBatch(int maxMsgsNumBatch) { + this.maxMsgsNumBatch = maxMsgsNumBatch; + } + + + public int getHaListenPort() { + return haListenPort; + } + + + public void setHaListenPort(int haListenPort) { + this.haListenPort = haListenPort; + } + + + public int getHaSendHeartbeatInterval() { + return haSendHeartbeatInterval; + } + + + public void setHaSendHeartbeatInterval(int haSendHeartbeatInterval) { + this.haSendHeartbeatInterval = haSendHeartbeatInterval; + } + + + public int getHaHousekeepingInterval() { + return haHousekeepingInterval; + } + + + public void setHaHousekeepingInterval(int haHousekeepingInterval) { + this.haHousekeepingInterval = haHousekeepingInterval; + } + + + public BrokerRole getBrokerRole() { + return brokerRole; + } + + + public void setBrokerRole(BrokerRole brokerRole) { + this.brokerRole = brokerRole; + } + + + public void setBrokerRole(String brokerRole) { + this.brokerRole = BrokerRole.valueOf(brokerRole); + } + + + public int getHaTransferBatchSize() { + return haTransferBatchSize; + } + + + public void setHaTransferBatchSize(int haTransferBatchSize) { + this.haTransferBatchSize = haTransferBatchSize; + } + + + public int getHaSlaveFallbehindMax() { + return haSlaveFallbehindMax; + } + + + public void setHaSlaveFallbehindMax(int haSlaveFallbehindMax) { + this.haSlaveFallbehindMax = haSlaveFallbehindMax; + } + + + public FlushDiskType getFlushDiskType() { + return flushDiskType; + } + + + public void setFlushDiskType(FlushDiskType flushDiskType) { + this.flushDiskType = flushDiskType; + } + + + public void setFlushDiskType(String type) { + this.flushDiskType = FlushDiskType.valueOf(type); + } + + + public int getSyncFlushTimeout() { + return syncFlushTimeout; + } + + + public void setSyncFlushTimeout(int syncFlushTimeout) { + this.syncFlushTimeout = syncFlushTimeout; + } + + + public String getHaMasterAddress() { + return haMasterAddress; + } + + + public void setHaMasterAddress(String haMasterAddress) { + this.haMasterAddress = haMasterAddress; + } + + + public String getMessageDelayLevel() { + return messageDelayLevel; + } + + + public void setMessageDelayLevel(String messageDelayLevel) { + this.messageDelayLevel = messageDelayLevel; + } + + + public long getFlushDelayOffsetInterval() { + return flushDelayOffsetInterval; + } + + + public void setFlushDelayOffsetInterval(long flushDelayOffsetInterval) { + this.flushDelayOffsetInterval = flushDelayOffsetInterval; + } + + + public boolean isCleanFileForciblyEnable() { + return cleanFileForciblyEnable; + } + + + public void setCleanFileForciblyEnable(boolean cleanFileForciblyEnable) { + this.cleanFileForciblyEnable = cleanFileForciblyEnable; + } + + + public boolean isMessageIndexSafe() { + return messageIndexSafe; + } + + + public void setMessageIndexSafe(boolean messageIndexSafe) { + this.messageIndexSafe = messageIndexSafe; + } + + + public boolean isFlushCommitLogTimed() { + return flushCommitLogTimed; + } + + + public void setFlushCommitLogTimed(boolean flushCommitLogTimed) { + this.flushCommitLogTimed = flushCommitLogTimed; + } + + + public String getStorePathRootDir() { + return storePathRootDir; + } + + + public void setStorePathRootDir(String storePathRootDir) { + this.storePathRootDir = storePathRootDir; + } +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/config/StorePathConfigHelper.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/config/StorePathConfigHelper.java new file mode 100644 index 000000000..ce392346d --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/config/StorePathConfigHelper.java @@ -0,0 +1,57 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store.config; + +import java.io.File; + + +public class StorePathConfigHelper { + + public static String getStorePathConsumeQueue(final String rootDir) { + return rootDir + File.separator + "consumequeue"; + } + + + public static String getStorePathIndex(final String rootDir) { + return rootDir + File.separator + "index"; + } + + + public static String getStoreCheckpoint(final String rootDir) { + return rootDir + File.separator + "checkpoint"; + } + + + public static String getAbortFile(final String rootDir) { + return rootDir + File.separator + "abort"; + } + + + public static String getDelayOffsetStorePath(final String rootDir) { + return rootDir + File.separator + "config" + File.separator + "delayOffset.json"; + } + + + public static String getTranStateTableStorePath(final String rootDir) { + return rootDir + File.separator + "transaction" + File.separator + "statetable"; + } + + + public static String getTranRedoLogStorePath(final String rootDir) { + return rootDir + File.separator + "transaction" + File.separator + "redolog"; + } + +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/ha/HAConnection.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/ha/HAConnection.java new file mode 100644 index 000000000..dc3ad1005 --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/ha/HAConnection.java @@ -0,0 +1,473 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store.ha; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.SelectionKey; +import java.nio.channels.Selector; +import java.nio.channels.SocketChannel; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.rocketmq.common.ServiceThread; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.remoting.common.RemotingUtil; +import com.alibaba.rocketmq.store.SelectMapedBufferResult; + + +/** + * HA服务，Master用来向Slave Push数据，并接收Slave应答 + * + * @author shijia.wxr + * @since 2013-7-21 + */ +public class HAConnection { + private static final Logger log = LoggerFactory.getLogger(LoggerName.StoreLoggerName); + private final HAService haService; + private final SocketChannel socketChannel; + private final String clientAddr; + private WriteSocketService writeSocketService; + private ReadSocketService readSocketService; + // Slave请求从哪里开始拉数据 + private volatile long slaveRequestOffset = -1; + // Slave收到数据后，应答Offset + private volatile long slaveAckOffset = -1; + + + public HAConnection(final HAService haService, final SocketChannel socketChannel) throws IOException { + this.haService = haService; + this.socketChannel = socketChannel; + this.clientAddr = this.socketChannel.socket().getRemoteSocketAddress().toString(); + this.socketChannel.configureBlocking(false); + this.socketChannel.socket().setSoLinger(false, -1); + this.socketChannel.socket().setTcpNoDelay(true); + this.socketChannel.socket().setReceiveBufferSize(1024 * 64); + this.socketChannel.socket().setSendBufferSize(1024 * 64); + this.writeSocketService = new WriteSocketService(this.socketChannel); + this.readSocketService = new ReadSocketService(this.socketChannel); + this.haService.getConnectionCount().incrementAndGet(); + } + + + /** + * 向Slave传输数据协议
+ * 从Slave接收数据协议 + */ + + public void start() { + this.readSocketService.start(); + this.writeSocketService.start(); + } + + + public void shutdown() { + this.writeSocketService.shutdown(true); + this.readSocketService.shutdown(true); + this.close(); + } + + + public void close() { + if (this.socketChannel != null) { + try { + this.socketChannel.close(); + } + catch (IOException e) { + HAConnection.log.error("", e); + } + } + } + + + public SocketChannel getSocketChannel() { + return socketChannel; + } + + /** + * 读取Slave请求，一般为push ack + * + * @author shijia.wxr + */ + class ReadSocketService extends ServiceThread { + private static final int ReadMaxBufferSize = 1024 * 1024; + private final Selector selector; + private final SocketChannel socketChannel; + private final ByteBuffer byteBufferRead = ByteBuffer.allocate(ReadMaxBufferSize); + private int processPostion = 0; + private volatile long lastReadTimestamp = System.currentTimeMillis(); + + + public ReadSocketService(final SocketChannel socketChannel) throws IOException { + this.selector = RemotingUtil.openSelector(); + this.socketChannel = socketChannel; + this.socketChannel.register(this.selector, SelectionKey.OP_READ); + // 线程自动回收，不需要被其他线程join + this.thread.setDaemon(true); + } + + + @Override + public void run() { + HAConnection.log.info(this.getServiceName() + " service started"); + + while (!this.isStoped()) { + try { + this.selector.select(1000); + boolean ok = this.processReadEvent(); + if (!ok) { + HAConnection.log.error("processReadEvent error"); + break; + } + + // 检测心跳间隔时间，超过则强制断开 + long interval = + HAConnection.this.haService.getDefaultMessageStore().getSystemClock().now() + - this.lastReadTimestamp; + if (interval > HAConnection.this.haService.getDefaultMessageStore() + .getMessageStoreConfig().getHaHousekeepingInterval()) { + log.warn("ha housekeeping, found this connection[" + HAConnection.this.clientAddr + + "] expired, " + interval); + break; + } + } + catch (Exception e) { + HAConnection.log.error(this.getServiceName() + " service has exception.", e); + break; + } + } + + this.makeStop(); + + // 避免内存泄露 + haService.removeConnection(HAConnection.this); + + // 只有读线程需要执行 + HAConnection.this.haService.getConnectionCount().decrementAndGet(); + + SelectionKey sk = this.socketChannel.keyFor(this.selector); + if (sk != null) { + sk.cancel(); + } + + try { + this.selector.close(); + this.socketChannel.close(); + } + catch (IOException e) { + HAConnection.log.error("", e); + } + + HAConnection.log.info(this.getServiceName() + " service end"); + } + + + private boolean processReadEvent() { + int readSizeZeroTimes = 0; + + if (!this.byteBufferRead.hasRemaining()) { + this.byteBufferRead.flip(); + this.processPostion = 0; + } + + while (this.byteBufferRead.hasRemaining()) { + try { + int readSize = this.socketChannel.read(this.byteBufferRead); + if (readSize > 0) { + readSizeZeroTimes = 0; + this.lastReadTimestamp = + HAConnection.this.haService.getDefaultMessageStore().getSystemClock().now(); + // 接收Slave上传的offset + if ((this.byteBufferRead.position() - this.processPostion) >= 8) { + int pos = this.byteBufferRead.position() - (this.byteBufferRead.position() % 8); + long readOffset = this.byteBufferRead.getLong(pos - 8); + this.processPostion = pos; + + // 处理Slave的请求 + HAConnection.this.slaveAckOffset = readOffset; + if (HAConnection.this.slaveRequestOffset < 0) { + HAConnection.this.slaveRequestOffset = readOffset; + log.info("slave[" + HAConnection.this.clientAddr + "] request offset " + + readOffset); + } + + // 通知前端线程 + HAConnection.this.haService.notifyTransferSome(HAConnection.this.slaveAckOffset); + } + } + else if (readSize == 0) { + if (++readSizeZeroTimes >= 3) { + break; + } + } + else { + log.error("read socket[" + HAConnection.this.clientAddr + "] < 0"); + return false; + } + } + catch (IOException e) { + log.error("processReadEvent exception", e); + return false; + } + } + + return true; + } + + + @Override + public String getServiceName() { + return ReadSocketService.class.getSimpleName(); + } + } + + /** + * 向Slave写入数据 + * + * @author shijia.wxr + */ + class WriteSocketService extends ServiceThread { + private final Selector selector; + private final SocketChannel socketChannel; + // 要传输的数据 + private final int HEADER_SIZE = 8 + 4; + private final ByteBuffer byteBufferHeader = ByteBuffer.allocate(HEADER_SIZE); + private long nextTransferFromWhere = -1; + private SelectMapedBufferResult selectMapedBufferResult; + private boolean lastWriteOver = true; + private long lastWriteTimestamp = System.currentTimeMillis(); + + + public WriteSocketService(final SocketChannel socketChannel) throws IOException { + this.selector = RemotingUtil.openSelector(); + this.socketChannel = socketChannel; + this.socketChannel.register(this.selector, SelectionKey.OP_WRITE); + this.thread.setDaemon(true); + } + + + @Override + public void run() { + HAConnection.log.info(this.getServiceName() + " service started"); + + while (!this.isStoped()) { + try { + this.selector.select(1000); + + if (-1 == HAConnection.this.slaveRequestOffset) { + Thread.sleep(10); + continue; + } + + // 第一次传输，需要计算从哪里开始 + // Slave如果本地没有数据，请求的Offset为0，那么master则从物理文件最后一个文件开始传送数据 + if (-1 == this.nextTransferFromWhere) { + if (0 == HAConnection.this.slaveRequestOffset) { + long masterOffset = + HAConnection.this.haService.getDefaultMessageStore().getCommitLog() + .getMaxOffset(); + masterOffset = + masterOffset + - (masterOffset % HAConnection.this.haService + .getDefaultMessageStore().getMessageStoreConfig() + .getMapedFileSizeCommitLog()); + + if (masterOffset < 0) { + masterOffset = 0; + } + + this.nextTransferFromWhere = masterOffset; + } + else { + this.nextTransferFromWhere = HAConnection.this.slaveRequestOffset; + } + + log.info("master transfer data from " + this.nextTransferFromWhere + " to slave[" + + HAConnection.this.clientAddr + "], and slave request " + + HAConnection.this.slaveRequestOffset); + } + + if (this.lastWriteOver) { + // 如果长时间没有发消息则尝试发心跳 + long interval = + HAConnection.this.haService.getDefaultMessageStore().getSystemClock().now() + - this.lastWriteTimestamp; + + if (interval > HAConnection.this.haService.getDefaultMessageStore() + .getMessageStoreConfig().getHaSendHeartbeatInterval()) { + // 向Slave发送心跳 + // Build Header + this.byteBufferHeader.position(0); + this.byteBufferHeader.limit(HEADER_SIZE); + this.byteBufferHeader.putLong(this.nextTransferFromWhere); + this.byteBufferHeader.putInt(0); + this.byteBufferHeader.flip(); + + this.lastWriteOver = this.transferData(); + if (!this.lastWriteOver) + continue; + } + } + // 继续传输 + else { + this.lastWriteOver = this.transferData(); + if (!this.lastWriteOver) + continue; + } + + // 传输数据, + // selectResult会赋值给this.selectMapedBufferResult，出现异常也会清理掉 + SelectMapedBufferResult selectResult = + HAConnection.this.haService.getDefaultMessageStore().getCommitLogData( + this.nextTransferFromWhere); + if (selectResult != null) { + int size = selectResult.getSize(); + if (size > HAConnection.this.haService.getDefaultMessageStore() + .getMessageStoreConfig().getHaTransferBatchSize()) { + size = + HAConnection.this.haService.getDefaultMessageStore() + .getMessageStoreConfig().getHaTransferBatchSize(); + } + + long thisOffset = this.nextTransferFromWhere; + this.nextTransferFromWhere += size; + + selectResult.getByteBuffer().limit(size); + this.selectMapedBufferResult = selectResult; + + // Build Header + this.byteBufferHeader.position(0); + this.byteBufferHeader.limit(HEADER_SIZE); + this.byteBufferHeader.putLong(thisOffset); + this.byteBufferHeader.putInt(size); + this.byteBufferHeader.flip(); + + this.lastWriteOver = this.transferData(); + } + else { + // 没有数据，等待通知 + HAConnection.this.haService.getWaitNotifyObject().allWaitForRunning(100); + } + } + catch (Exception e) { + // 只要抛出异常，一般是网络发生错误，连接必须断开，并清理资源 + HAConnection.log.error(this.getServiceName() + " service has exception.", e); + break; + } + } + + // 清理资源 + if (this.selectMapedBufferResult != null) { + this.selectMapedBufferResult.release(); + } + + this.makeStop(); + + // 避免内存泄露 + haService.removeConnection(HAConnection.this); + + SelectionKey sk = this.socketChannel.keyFor(this.selector); + if (sk != null) { + sk.cancel(); + } + + try { + this.selector.close(); + this.socketChannel.close(); + } + catch (IOException e) { + HAConnection.log.error("", e); + } + + HAConnection.log.info(this.getServiceName() + " service end"); + } + + + /** + * 表示是否传输完成 + */ + private boolean transferData() throws Exception { + int writeSizeZeroTimes = 0; + // Write Header + while (this.byteBufferHeader.hasRemaining()) { + int writeSize = this.socketChannel.write(this.byteBufferHeader); + if (writeSize > 0) { + writeSizeZeroTimes = 0; + this.lastWriteTimestamp = + HAConnection.this.haService.getDefaultMessageStore().getSystemClock().now(); + } + else if (writeSize == 0) { + if (++writeSizeZeroTimes >= 3) { + break; + } + } + else { + throw new Exception("ha master write header error < 0"); + } + } + + if (null == this.selectMapedBufferResult) { + return !this.byteBufferHeader.hasRemaining(); + } + + writeSizeZeroTimes = 0; + + // Write Body + if (!this.byteBufferHeader.hasRemaining()) { + while (this.selectMapedBufferResult.getByteBuffer().hasRemaining()) { + int writeSize = this.socketChannel.write(this.selectMapedBufferResult.getByteBuffer()); + if (writeSize > 0) { + writeSizeZeroTimes = 0; + this.lastWriteTimestamp = + HAConnection.this.haService.getDefaultMessageStore().getSystemClock().now(); + } + else if (writeSize == 0) { + if (++writeSizeZeroTimes >= 3) { + break; + } + } + else { + throw new Exception("ha master write body error < 0"); + } + } + } + + boolean result = + !this.byteBufferHeader.hasRemaining() + && !this.selectMapedBufferResult.getByteBuffer().hasRemaining(); + + if (!this.selectMapedBufferResult.getByteBuffer().hasRemaining()) { + this.selectMapedBufferResult.release(); + this.selectMapedBufferResult = null; + } + + return result; + } + + + @Override + public String getServiceName() { + return WriteSocketService.class.getSimpleName(); + } + + + @Override + public void shutdown() { + super.shutdown(); + } + } +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/ha/HAService.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/ha/HAService.java new file mode 100644 index 000000000..583013b8d --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/ha/HAService.java @@ -0,0 +1,693 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store.ha; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.nio.ByteBuffer; +import java.nio.channels.ClosedChannelException; +import java.nio.channels.SelectionKey; +import java.nio.channels.Selector; +import java.nio.channels.ServerSocketChannel; +import java.nio.channels.SocketChannel; +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.rocketmq.common.ServiceThread; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.remoting.common.RemotingUtil; +import com.alibaba.rocketmq.store.CommitLog.GroupCommitRequest; +import com.alibaba.rocketmq.store.DefaultMessageStore; + + +/** + * HA服务，负责同步双写，异步复制功能 + * + * @author shijia.wxr + * @since 2013-7-21 + */ +public class HAService { + private static final Logger log = LoggerFactory.getLogger(LoggerName.StoreLoggerName); + // 客户端连接计数 + private final AtomicInteger connectionCount = new AtomicInteger(0); + // 存储客户端连接 + private final List connectionList = new LinkedList(); + // 接收新的Socket连接 + private final AcceptSocketService acceptSocketService; + // 顶层存储对象 + private final DefaultMessageStore defaultMessageStore; + // 异步通知 + private final WaitNotifyObject waitNotifyObject = new WaitNotifyObject(); + // 写入到Slave的最大Offset + private final AtomicLong push2SlaveMaxOffset = new AtomicLong(0); + // 主从复制通知服务 + private final GroupTransferService groupTransferService; + // Slave订阅对象 + private final HAClient haClient; + + + public HAService(final DefaultMessageStore defaultMessageStore) throws IOException { + this.defaultMessageStore = defaultMessageStore; + this.acceptSocketService = + new AcceptSocketService(defaultMessageStore.getMessageStoreConfig().getHaListenPort()); + this.groupTransferService = new GroupTransferService(); + this.haClient = new HAClient(); + } + + + public void updateMasterAddress(final String newAddr) { + if (this.haClient != null) { + this.haClient.updateMasterAddress(newAddr); + } + } + + + public void putRequest(final GroupCommitRequest request) { + this.groupTransferService.putRequest(request); + } + + + /** + * 判断主从之间数据传输是否正常 + * + * @return + */ + public boolean isSlaveOK(final long masterPutWhere) { + boolean result = this.connectionCount.get() > 0; + result = + result + && ((masterPutWhere - this.push2SlaveMaxOffset.get()) < this.defaultMessageStore + .getMessageStoreConfig().getHaSlaveFallbehindMax()); + return result; + } + + + /** + * 通知复制了部分数据 + */ + public void notifyTransferSome(final long offset) { + for (long value = this.push2SlaveMaxOffset.get(); offset > value;) { + boolean ok = this.push2SlaveMaxOffset.compareAndSet(value, offset); + if (ok) { + this.groupTransferService.notifyTransferSome(); + break; + } + else { + value = this.push2SlaveMaxOffset.get(); + } + } + } + + + public AtomicInteger getConnectionCount() { + return connectionCount; + } + + + // public void notifyTransferSome() { + // this.groupTransferService.notifyTransferSome(); + // } + + public void start() { + this.acceptSocketService.beginAccept(); + this.acceptSocketService.start(); + this.groupTransferService.start(); + this.haClient.start(); + } + + + public void addConnection(final HAConnection conn) { + synchronized (this.connectionList) { + this.connectionList.add(conn); + } + } + + + public void removeConnection(final HAConnection conn) { + synchronized (this.connectionList) { + this.connectionList.remove(conn); + } + } + + + public void shutdown() { + this.haClient.shutdown(); + this.acceptSocketService.shutdown(true); + this.destroyConnections(); + this.groupTransferService.shutdown(); + } + + + public void destroyConnections() { + synchronized (this.connectionList) { + for (HAConnection c : this.connectionList) { + c.shutdown(); + } + + this.connectionList.clear(); + } + } + + + public DefaultMessageStore getDefaultMessageStore() { + return defaultMessageStore; + } + + + public WaitNotifyObject getWaitNotifyObject() { + return waitNotifyObject; + } + + class AcceptSocketService extends ServiceThread { + private ServerSocketChannel serverSocketChannel; + private Selector selector; + private SocketAddress socketAddressListen; + + + public AcceptSocketService(final int port) { + this.socketAddressListen = new InetSocketAddress(port); + } + + + public void beginAccept() { + try { + this.serverSocketChannel = ServerSocketChannel.open(); + this.selector = RemotingUtil.openSelector(); + this.serverSocketChannel.socket().setReuseAddress(true); + this.serverSocketChannel.socket().bind(this.socketAddressListen); + this.serverSocketChannel.configureBlocking(false); + this.serverSocketChannel.register(this.selector, SelectionKey.OP_ACCEPT); + } + catch (Exception e) { + log.error("beginAccept exception", e); + } + } + + + @Override + public void run() { + log.info(this.getServiceName() + " service started"); + + while (!this.isStoped()) { + try { + this.selector.select(1000); + Set selected = this.selector.selectedKeys(); + if (selected != null) { + for (SelectionKey k : selected) { + if ((k.readyOps() & SelectionKey.OP_ACCEPT) != 0) { + SocketChannel sc = ((ServerSocketChannel) k.channel()).accept(); + if (sc != null) { + HAService.log.info("HAService receive new connection, " + + sc.socket().getRemoteSocketAddress()); + + try { + HAConnection conn = new HAConnection(HAService.this, sc); + conn.start(); + HAService.this.addConnection(conn); + } + catch (Exception e) { + log.error("new HAConnection exception", e); + sc.close(); + } + } + } + else { + log.warn("Unexpected ops in select " + k.readyOps()); + } + } + + selected.clear(); + } + + } + catch (Exception e) { + log.error(this.getServiceName() + " service has exception.", e); + } + } + + log.error(this.getServiceName() + " service end"); + } + + + @Override + public String getServiceName() { + return AcceptSocketService.class.getSimpleName(); + } + } + + /** + * GroupTransferService Service + */ + class GroupTransferService extends ServiceThread { + // 异步通知 + private final WaitNotifyObject notifyTransferObject = new WaitNotifyObject(); + private volatile List requestsWrite = new ArrayList(); + private volatile List requestsRead = new ArrayList(); + + + public void putRequest(final GroupCommitRequest request) { + synchronized (this) { + this.requestsWrite.add(request); + if (!this.hasNotified) { + this.hasNotified = true; + this.notify(); + + // TODO 这里要Notify两个线程 1、GroupTransferService + // 2、WriteSocketService + // 在调用putRequest后，已经Notify了WriteSocketService + } + } + } + + + public void notifyTransferSome() { + this.notifyTransferObject.wakeup(); + } + + + private void swapRequests() { + List tmp = this.requestsWrite; + this.requestsWrite = this.requestsRead; + this.requestsRead = tmp; + } + + + private void doWaitTransfer() { + if (!this.requestsRead.isEmpty()) { + for (GroupCommitRequest req : this.requestsRead) { + boolean transferOK = HAService.this.push2SlaveMaxOffset.get() >= req.getNextOffset(); + for (int i = 0; !transferOK && i < 5;) { + this.notifyTransferObject.waitForRunning(1000); + transferOK = HAService.this.push2SlaveMaxOffset.get() >= req.getNextOffset(); + } + + if (!transferOK) { + log.warn("transfer messsage to slave timeout, " + req.getNextOffset()); + } + + req.wakeupCustomer(transferOK); + } + + this.requestsRead.clear(); + } + } + + + public void run() { + log.info(this.getServiceName() + " service started"); + + while (!this.isStoped()) { + try { + this.waitForRunning(0); + this.doWaitTransfer(); + } + catch (Exception e) { + log.warn(this.getServiceName() + " service has exception. ", e); + } + } + + log.info(this.getServiceName() + " service end"); + } + + + @Override + protected void onWaitEnd() { + this.swapRequests(); + } + + + @Override + public String getServiceName() { + return GroupTransferService.class.getSimpleName(); + } + } + + class HAClient extends ServiceThread { + private static final int ReadMaxBufferSize = 1024 * 1024 * 4; + // 主节点IP:PORT + private final AtomicReference masterAddress = new AtomicReference(); + // 向Master汇报Slave最大Offset + private final ByteBuffer reportOffset = ByteBuffer.allocate(8); + private SocketChannel socketChannel; + private Selector selector; + private long lastWriteTimestamp = System.currentTimeMillis(); + // Slave向Master汇报Offset，汇报到哪里 + private long currentReportedOffset = 0; + private int dispatchPostion = 0; + // 从Master接收数据Buffer + private ByteBuffer byteBufferRead = ByteBuffer.allocate(ReadMaxBufferSize); + private ByteBuffer byteBufferBackup = ByteBuffer.allocate(ReadMaxBufferSize); + + + public HAClient() throws IOException { + this.selector = RemotingUtil.openSelector(); + } + + + public void updateMasterAddress(final String newAddr) { + String currentAddr = this.masterAddress.get(); + if (currentAddr == null || !currentAddr.equals(newAddr)) { + this.masterAddress.set(newAddr); + log.info("update master address, OLD: " + currentAddr + " NEW: " + newAddr); + } + } + + + private boolean isTimeToReportOffset() { + long interval = + HAService.this.defaultMessageStore.getSystemClock().now() - this.lastWriteTimestamp; + boolean needHeart = + (interval > HAService.this.defaultMessageStore.getMessageStoreConfig() + .getHaSendHeartbeatInterval()); + + return needHeart; + } + + + private boolean reportSlaveMaxOffset(final long maxOffset) { + this.reportOffset.position(0); + this.reportOffset.limit(8); + this.reportOffset.putLong(maxOffset); + this.reportOffset.position(0); + this.reportOffset.limit(8); + + for (int i = 0; i < 3 && this.reportOffset.hasRemaining(); i++) { + try { + this.socketChannel.write(this.reportOffset); + } + catch (IOException e) { + log.error(this.getServiceName() + + "reportSlaveMaxOffset this.socketChannel.write exception", e); + return false; + } + } + + return !this.reportOffset.hasRemaining(); + } + + + // private void reallocateByteBuffer() { + // ByteBuffer bb = ByteBuffer.allocate(ReadMaxBufferSize); + // int remain = this.byteBufferRead.limit() - this.dispatchPostion; + // bb.put(this.byteBufferRead.array(), this.dispatchPostion, remain); + // this.dispatchPostion = 0; + // this.byteBufferRead = bb; + // } + + /** + * Buffer满了以后，重新整理一次 + */ + private void reallocateByteBuffer() { + int remain = ReadMaxBufferSize - this.dispatchPostion; + if (remain > 0) { + this.byteBufferRead.position(this.dispatchPostion); + + this.byteBufferBackup.position(0); + this.byteBufferBackup.limit(ReadMaxBufferSize); + this.byteBufferBackup.put(this.byteBufferRead); + } + + this.swapByteBuffer(); + + this.byteBufferRead.position(remain); + this.byteBufferRead.limit(ReadMaxBufferSize); + this.dispatchPostion = 0; + } + + + private void swapByteBuffer() { + ByteBuffer tmp = this.byteBufferRead; + this.byteBufferRead = this.byteBufferBackup; + this.byteBufferBackup = tmp; + } + + + private boolean processReadEvent() { + int readSizeZeroTimes = 0; + while (this.byteBufferRead.hasRemaining()) { + try { + int readSize = this.socketChannel.read(this.byteBufferRead); + if (readSize > 0) { + lastWriteTimestamp = HAService.this.defaultMessageStore.getSystemClock().now(); + readSizeZeroTimes = 0; + boolean result = this.dispatchReadRequest(); + if (!result) { + log.error("HAClient, dispatchReadRequest error"); + return false; + } + } + else if (readSize == 0) { + if (++readSizeZeroTimes >= 3) { + break; + } + } + else { + // TODO ERROR + log.info("HAClient, processReadEvent read socket < 0"); + return false; + } + } + catch (IOException e) { + log.info("HAClient, processReadEvent read socket exception", e); + return false; + } + } + + return true; + } + + + private boolean dispatchReadRequest() { + final int MSG_HEADER_SIZE = 8 + 4; // phyoffset + size + int readSocketPos = this.byteBufferRead.position(); + + while (true) { + int diff = this.byteBufferRead.position() - this.dispatchPostion; + if (diff >= MSG_HEADER_SIZE) { + long masterPhyOffset = this.byteBufferRead.getLong(this.dispatchPostion); + int bodySize = this.byteBufferRead.getInt(this.dispatchPostion + 8); + + long slavePhyOffset = HAService.this.defaultMessageStore.getMaxPhyOffset(); + + // 发生重大错误 + if (slavePhyOffset != 0) { + if (slavePhyOffset != masterPhyOffset) { + log.error("master pushed offset not equal the max phy offset in slave, SLAVE: " + + slavePhyOffset + " MASTER: " + masterPhyOffset); + return false; + } + } + + // 可以凑够一个请求 + if (diff >= (MSG_HEADER_SIZE + bodySize)) { + byte[] bodyData = new byte[bodySize]; + this.byteBufferRead.position(this.dispatchPostion + MSG_HEADER_SIZE); + this.byteBufferRead.get(bodyData); + + // TODO 结果是否需要处理，暂时不处理 + HAService.this.defaultMessageStore.appendToCommitLog(masterPhyOffset, bodyData); + + this.byteBufferRead.position(readSocketPos); + this.dispatchPostion += MSG_HEADER_SIZE + bodySize; + + if (!reportSlaveMaxOffsetPlus()) { + return false; + } + + continue; + } + } + + if (!this.byteBufferRead.hasRemaining()) { + this.reallocateByteBuffer(); + } + + break; + } + + return true; + } + + + private boolean reportSlaveMaxOffsetPlus() { + boolean result = true; + // 只要本地有更新，就汇报最大物理Offset + long currentPhyOffset = HAService.this.defaultMessageStore.getMaxPhyOffset(); + if (currentPhyOffset > this.currentReportedOffset) { + this.currentReportedOffset = currentPhyOffset; + result = this.reportSlaveMaxOffset(this.currentReportedOffset); + if (!result) { + this.closeMaster(); + log.error("HAClient, reportSlaveMaxOffset error, " + this.currentReportedOffset); + } + } + + return result; + } + + + private boolean connectMaster() throws ClosedChannelException { + if (null == socketChannel) { + String addr = this.masterAddress.get(); + if (addr != null) { + + SocketAddress socketAddress = RemotingUtil.string2SocketAddress(addr); + if (socketAddress != null) { + this.socketChannel = RemotingUtil.connect(socketAddress); + if (this.socketChannel != null) { + this.socketChannel.register(this.selector, SelectionKey.OP_READ); + } + } + } + + // 每次连接时，要重新拿到最大的Offset + this.currentReportedOffset = HAService.this.defaultMessageStore.getMaxPhyOffset(); + + this.lastWriteTimestamp = System.currentTimeMillis(); + } + + return this.socketChannel != null; + } + + + private void closeMaster() { + if (null != this.socketChannel) { + try { + + SelectionKey sk = this.socketChannel.keyFor(this.selector); + if (sk != null) { + sk.cancel(); + } + + this.socketChannel.close(); + + this.socketChannel = null; + } + catch (IOException e) { + log.warn("closeMaster exception. ", e); + } + + this.lastWriteTimestamp = 0; + this.dispatchPostion = 0; + + this.byteBufferBackup.position(0); + this.byteBufferBackup.limit(ReadMaxBufferSize); + + this.byteBufferRead.position(0); + this.byteBufferRead.limit(ReadMaxBufferSize); + } + } + + + @Override + public void run() { + log.info(this.getServiceName() + " service started"); + + while (!this.isStoped()) { + try { + if (this.connectMaster()) { + // 先汇报最大物理Offset || 定时心跳方式汇报 + if (this.isTimeToReportOffset()) { + boolean result = this.reportSlaveMaxOffset(this.currentReportedOffset); + if (!result) { + this.closeMaster(); + } + } + + // 等待应答 + this.selector.select(1000); + + // 接收数据 + boolean ok = this.processReadEvent(); + if (!ok) { + this.closeMaster(); + } + + // 只要本地有更新，就汇报最大物理Offset + if (!reportSlaveMaxOffsetPlus()) { + continue; + } + + // 检查Master的反向心跳 + long interval = + HAService.this.getDefaultMessageStore().getSystemClock().now() + - this.lastWriteTimestamp; + if (interval > HAService.this.getDefaultMessageStore().getMessageStoreConfig() + .getHaHousekeepingInterval()) { + log.warn("HAClient, housekeeping, found this connection[" + this.masterAddress + + "] expired, " + interval); + this.closeMaster(); + log.warn("HAClient, master not response some time, so close connection"); + } + } + else { + this.waitForRunning(1000 * 5); + } + } + catch (Exception e) { + log.warn(this.getServiceName() + " service has exception. ", e); + this.waitForRunning(1000 * 5); + } + } + + log.info(this.getServiceName() + " service end"); + } + + + // + // private void disableWriteFlag() { + // if (this.socketChannel != null) { + // SelectionKey sk = this.socketChannel.keyFor(this.selector); + // if (sk != null) { + // int ops = sk.interestOps(); + // ops &= ~SelectionKey.OP_WRITE; + // sk.interestOps(ops); + // } + // } + // } + // + // + // private void enableWriteFlag() { + // if (this.socketChannel != null) { + // SelectionKey sk = this.socketChannel.keyFor(this.selector); + // if (sk != null) { + // int ops = sk.interestOps(); + // ops |= SelectionKey.OP_WRITE; + // sk.interestOps(ops); + // } + // } + // } + + @Override + public String getServiceName() { + return HAClient.class.getSimpleName(); + } + } + + + public AtomicLong getPush2SlaveMaxOffset() { + return push2SlaveMaxOffset; + } +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/ha/WaitNotifyObject.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/ha/WaitNotifyObject.java new file mode 100644 index 000000000..277853a37 --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/ha/WaitNotifyObject.java @@ -0,0 +1,115 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store.ha; + +import java.util.HashMap; + + +/** + * 用来做线程之间异步通知 + * + * @author shijia.wxr + * @since 2013-7-21 + */ +public class WaitNotifyObject { + // 是否已经被Notify过，广播模式 + protected final HashMap waitingThreadTable = + new HashMap(16); + // 是否已经被Notify过 + protected volatile boolean hasNotified = false; + + + public void wakeup() { + synchronized (this) { + if (!this.hasNotified) { + this.hasNotified = true; + this.notify(); + } + } + } + + + protected void waitForRunning(long interval) { + synchronized (this) { + if (this.hasNotified) { + this.hasNotified = false; + this.onWaitEnd(); + return; + } + + try { + this.wait(interval); + } + catch (InterruptedException e) { + e.printStackTrace(); + } + finally { + this.hasNotified = false; + this.onWaitEnd(); + } + } + } + + + protected void onWaitEnd() { + } + + + /** + * 广播方式唤醒 + */ + public void wakeupAll() { + synchronized (this) { + boolean needNotify = false; + + for (Boolean value : this.waitingThreadTable.values()) { + needNotify = needNotify || !value; + value = true; + } + + if (needNotify) { + this.notifyAll(); + } + } + } + + + /** + * 多个线程调用wait + */ + public void allWaitForRunning(long interval) { + long currentThreadId = Thread.currentThread().getId(); + synchronized (this) { + Boolean notified = this.waitingThreadTable.get(currentThreadId); + if (notified != null && notified) { + this.waitingThreadTable.put(currentThreadId, false); + this.onWaitEnd(); + return; + } + + try { + this.wait(interval); + } + catch (InterruptedException e) { + e.printStackTrace(); + } + finally { + this.waitingThreadTable.put(currentThreadId, false); + this.onWaitEnd(); + } + } + } +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/index/IndexFile.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/index/IndexFile.java new file mode 100644 index 000000000..036c7b7c1 --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/index/IndexFile.java @@ -0,0 +1,322 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store.index; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.MappedByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.channels.FileLock; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.store.MapedFile; + + +/** + * 存储具体消息索引信息的文件 + * + * @author shijia.wxr + * @since 2013-7-21 + */ +public class IndexFile { + private static final Logger log = LoggerFactory.getLogger(LoggerName.StoreLoggerName); + private static int HASH_SLOT_SIZE = 4; + private static int INDEX_SIZE = 20; + private static int INVALID_INDEX = 0; + private final int hashSlotNum; + private final int indexNum; + private final MapedFile mapedFile; + private final FileChannel fileChannel; + private final MappedByteBuffer mappedByteBuffer; + private final IndexHeader indexHeader; + + + public IndexFile(final String fileName, final int hashSlotNum, final int indexNum, + final long endPhyOffset, final long endTimestamp) throws IOException { + int fileTotalSize = + IndexHeader.INDEX_HEADER_SIZE + (hashSlotNum * HASH_SLOT_SIZE) + (indexNum * INDEX_SIZE); + this.mapedFile = new MapedFile(fileName, fileTotalSize); + this.fileChannel = this.mapedFile.getFileChannel(); + this.mappedByteBuffer = this.mapedFile.getMappedByteBuffer(); + this.hashSlotNum = hashSlotNum; + this.indexNum = indexNum; + + ByteBuffer byteBuffer = this.mappedByteBuffer.slice(); + this.indexHeader = new IndexHeader(byteBuffer); + + if (endPhyOffset > 0) { + this.indexHeader.setBeginPhyOffset(endPhyOffset); + this.indexHeader.setEndPhyOffset(endPhyOffset); + } + + if (endTimestamp > 0) { + this.indexHeader.setBeginTimestamp(endTimestamp); + this.indexHeader.setEndTimestamp(endTimestamp); + } + } + + + public String getFileName() { + return this.mapedFile.getFileName(); + } + + + public void load() { + this.indexHeader.load(); + } + + + public void flush() { + long beginTime = System.currentTimeMillis(); + if (this.mapedFile.hold()) { + this.indexHeader.updateByteBuffer(); + this.mappedByteBuffer.force(); + this.mapedFile.release(); + log.info("flush index file eclipse time(ms) " + (System.currentTimeMillis() - beginTime)); + } + } + + + /** + * 当前索引文件是否写满 + */ + public boolean isWriteFull() { + return this.indexHeader.getIndexCount() >= this.indexNum; + } + + + public boolean destroy(final long intervalForcibly) { + return this.mapedFile.destroy(intervalForcibly); + } + + + /** + * 如果返回false，表示需要创建新的索引文件 + */ + public boolean putKey(final String key, final long phyOffset, final long storeTimestamp) { + if (this.indexHeader.getIndexCount() < this.indexNum) { + int keyHash = indexKeyHashMethod(key); + int slotPos = keyHash % this.hashSlotNum; + int absSlotPos = IndexHeader.INDEX_HEADER_SIZE + slotPos * HASH_SLOT_SIZE; + + FileLock fileLock = null; + + try { + // TODO 是否是读写锁 + // fileLock = this.fileChannel.lock(absSlotPos, HASH_SLOT_SIZE, + // false); + int slotValue = this.mappedByteBuffer.getInt(absSlotPos); + if (slotValue <= INVALID_INDEX || slotValue > this.indexHeader.getIndexCount()) { + slotValue = INVALID_INDEX; + } + + long timeDiff = storeTimestamp - this.indexHeader.getBeginTimestamp(); + if (this.indexHeader.getBeginTimestamp() <= 0) { + timeDiff = 0; + } + else if (timeDiff > Integer.MAX_VALUE) { + timeDiff = Integer.MAX_VALUE; + } + else if (timeDiff < 0) { + timeDiff = 0; + } + + // 时间差存储单位由毫秒改为秒 + timeDiff = timeDiff / 1000; + + int absIndexPos = + IndexHeader.INDEX_HEADER_SIZE + this.hashSlotNum * HASH_SLOT_SIZE + + this.indexHeader.getIndexCount() * INDEX_SIZE; + + // 写入真正索引 + this.mappedByteBuffer.putInt(absIndexPos, keyHash); + this.mappedByteBuffer.putLong(absIndexPos + 4, phyOffset); + this.mappedByteBuffer.putInt(absIndexPos + 4 + 8, (int) timeDiff); + this.mappedByteBuffer.putInt(absIndexPos + 4 + 8 + 4, slotValue); + + // 更新哈希槽 + this.mappedByteBuffer.putInt(absSlotPos, this.indexHeader.getIndexCount()); + + // 第一次写入 + if (this.indexHeader.getIndexCount() <= 1) { + this.indexHeader.setBeginPhyOffset(phyOffset); + this.indexHeader.setBeginTimestamp(storeTimestamp); + } + + this.indexHeader.incHashSlotCount(); + this.indexHeader.incIndexCount(); + this.indexHeader.setEndPhyOffset(phyOffset); + this.indexHeader.setEndTimestamp(storeTimestamp); + + return true; + } + catch (Exception e) { + log.error("putKey exception, Key: " + key + " KeyHashCode: " + key.hashCode(), e); + } + finally { + if (fileLock != null) { + try { + fileLock.release(); + } + catch (IOException e) { + e.printStackTrace(); + } + } + } + } + else { + log.warn("putKey index count " + this.indexHeader.getIndexCount() + " index max num " + + this.indexNum); + } + + return false; + } + + + public long getBeginTimestamp() { + return this.indexHeader.getBeginTimestamp(); + } + + + public long getEndTimestamp() { + return this.indexHeader.getEndTimestamp(); + } + + + public long getEndPhyOffset() { + return this.indexHeader.getEndPhyOffset(); + } + + + /** + * 时间区间是否匹配 + */ + public boolean isTimeMatched(final long begin, final long end) { + boolean result = + begin < this.indexHeader.getBeginTimestamp() && end > this.indexHeader.getEndTimestamp(); + + result = + result + || (begin >= this.indexHeader.getBeginTimestamp() && begin <= this.indexHeader + .getEndTimestamp()); + + result = + result + || (end >= this.indexHeader.getBeginTimestamp() && end <= this.indexHeader + .getEndTimestamp()); + return result; + } + + + // 返回值是大于0 + public int indexKeyHashMethod(final String key) { + int keyHash = key.hashCode(); + int keyHashPositive = Math.abs(keyHash); + if (keyHashPositive < 0) + keyHashPositive = 0; + return keyHashPositive; + } + + + /** + * 前提：入参时间区间在调用前已经匹配了当前索引文件的起始结束时间 + */ + public void selectPhyOffset(final List phyOffsets, final String key, final int maxNum, + final long begin, final long end, boolean lock) { + if (this.mapedFile.hold()) { + int keyHash = indexKeyHashMethod(key); + int slotPos = keyHash % this.hashSlotNum; + int absSlotPos = IndexHeader.INDEX_HEADER_SIZE + slotPos * HASH_SLOT_SIZE; + + FileLock fileLock = null; + try { + if (lock) { + // fileLock = this.fileChannel.lock(absSlotPos, + // HASH_SLOT_SIZE, true); + } + + int slotValue = this.mappedByteBuffer.getInt(absSlotPos); + // if (fileLock != null) { + // fileLock.release(); + // fileLock = null; + // } + + if (slotValue <= INVALID_INDEX || slotValue > this.indexHeader.getIndexCount() + || this.indexHeader.getIndexCount() <= 1) { + // TODO NOTFOUND + } + else { + for (int nextIndexToRead = slotValue;;) { + if (phyOffsets.size() >= maxNum) { + break; + } + + int absIndexPos = + IndexHeader.INDEX_HEADER_SIZE + this.hashSlotNum * HASH_SLOT_SIZE + + nextIndexToRead * INDEX_SIZE; + + int keyHashRead = this.mappedByteBuffer.getInt(absIndexPos); + long phyOffsetRead = this.mappedByteBuffer.getLong(absIndexPos + 4); + int timeDiff = this.mappedByteBuffer.getInt(absIndexPos + 4 + 8); + int prevIndexRead = this.mappedByteBuffer.getInt(absIndexPos + 4 + 8 + 4); + + // 读到了未知数据 + if (timeDiff < 0) { + break; + } + + // 时间差存储的是秒，再还原为毫秒 + timeDiff *= 1000; + + long timeRead = this.indexHeader.getBeginTimestamp() + timeDiff; + boolean timeMatched = (timeRead >= begin) && (timeRead <= end); + + if (keyHash == keyHashRead && timeMatched) { + phyOffsets.add(phyOffsetRead); + } + + if (prevIndexRead <= INVALID_INDEX + || prevIndexRead > this.indexHeader.getIndexCount() + || prevIndexRead == nextIndexToRead || timeRead < begin) { + break; + } + + nextIndexToRead = prevIndexRead; + } + } + } + catch (Exception e) { + log.error("selectPhyOffset exception ", e); + } + finally { + if (fileLock != null) { + try { + fileLock.release(); + } + catch (IOException e) { + e.printStackTrace(); + } + } + + this.mapedFile.release(); + } + } + } +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/index/IndexHeader.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/index/IndexHeader.java new file mode 100644 index 000000000..909d94460 --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/index/IndexHeader.java @@ -0,0 +1,144 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store.index; + +import java.nio.ByteBuffer; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + + +/** + * 索引文件头 + * + * @author shijia.wxr + * @since 2013-7-21 + */ +public class IndexHeader { + public static final int INDEX_HEADER_SIZE = 40; + private static int BEGINTIMESTAMP_INDEX = 0; + private static int ENDTIMESTAMP_INDEX = 8; + private static int BEGINPHYOFFSET_INDEX = 16; + private static int ENDPHYOFFSET_INDEX = 24; + private static int HASHSLOTCOUNT_INDEX = 32; + private static int INDEXCOUNT_INDEX = 36; + private final ByteBuffer byteBuffer; + private AtomicLong beginTimestamp = new AtomicLong(0); + private AtomicLong endTimestamp = new AtomicLong(0); + private AtomicLong beginPhyOffset = new AtomicLong(0); + private AtomicLong endPhyOffset = new AtomicLong(0); + private AtomicInteger hashSlotCount = new AtomicInteger(0); + // 第一个索引是无效索引 + private AtomicInteger indexCount = new AtomicInteger(1); + + + public IndexHeader(final ByteBuffer byteBuffer) { + this.byteBuffer = byteBuffer; + } + + + public void load() { + this.beginTimestamp.set(byteBuffer.getLong(BEGINTIMESTAMP_INDEX)); + this.endTimestamp.set(byteBuffer.getLong(ENDTIMESTAMP_INDEX)); + this.beginPhyOffset.set(byteBuffer.getLong(BEGINPHYOFFSET_INDEX)); + this.endPhyOffset.set(byteBuffer.getLong(ENDPHYOFFSET_INDEX)); + + this.hashSlotCount.set(byteBuffer.getInt(HASHSLOTCOUNT_INDEX)); + this.indexCount.set(byteBuffer.getInt(INDEXCOUNT_INDEX)); + + if (this.indexCount.get() <= 0) { + this.indexCount.set(1); + } + } + + + /** + * 更新byteBuffer + */ + public void updateByteBuffer() { + this.byteBuffer.putLong(BEGINTIMESTAMP_INDEX, this.beginTimestamp.get()); + this.byteBuffer.putLong(ENDTIMESTAMP_INDEX, this.endTimestamp.get()); + this.byteBuffer.putLong(BEGINPHYOFFSET_INDEX, this.beginPhyOffset.get()); + this.byteBuffer.putLong(ENDPHYOFFSET_INDEX, this.endPhyOffset.get()); + this.byteBuffer.putInt(HASHSLOTCOUNT_INDEX, this.hashSlotCount.get()); + this.byteBuffer.putInt(INDEXCOUNT_INDEX, this.indexCount.get()); + } + + + public long getBeginTimestamp() { + return beginTimestamp.get(); + } + + + public void setBeginTimestamp(long beginTimestamp) { + this.beginTimestamp.set(beginTimestamp); + this.byteBuffer.putLong(BEGINTIMESTAMP_INDEX, beginTimestamp); + } + + + public long getEndTimestamp() { + return endTimestamp.get(); + } + + + public void setEndTimestamp(long endTimestamp) { + this.endTimestamp.set(endTimestamp); + this.byteBuffer.putLong(ENDTIMESTAMP_INDEX, endTimestamp); + } + + + public long getBeginPhyOffset() { + return beginPhyOffset.get(); + } + + + public void setBeginPhyOffset(long beginPhyOffset) { + this.beginPhyOffset.set(beginPhyOffset); + this.byteBuffer.putLong(BEGINPHYOFFSET_INDEX, beginPhyOffset); + } + + + public long getEndPhyOffset() { + return endPhyOffset.get(); + } + + + public void setEndPhyOffset(long endPhyOffset) { + this.endPhyOffset.set(endPhyOffset); + this.byteBuffer.putLong(ENDPHYOFFSET_INDEX, endPhyOffset); + } + + + public AtomicInteger getHashSlotCount() { + return hashSlotCount; + } + + + public void incHashSlotCount() { + int value = this.hashSlotCount.incrementAndGet(); + this.byteBuffer.putInt(HASHSLOTCOUNT_INDEX, value); + } + + + public int getIndexCount() { + return indexCount.get(); + } + + + public void incIndexCount() { + int value = this.indexCount.incrementAndGet(); + this.byteBuffer.putInt(INDEXCOUNT_INDEX, value); + } +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/index/IndexService.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/index/IndexService.java new file mode 100644 index 000000000..fad93eb7a --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/index/IndexService.java @@ -0,0 +1,443 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store.index; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.rocketmq.common.ServiceThread; +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.common.message.MessageConst; +import com.alibaba.rocketmq.common.sysflag.MessageSysFlag; +import com.alibaba.rocketmq.store.DefaultMessageStore; +import com.alibaba.rocketmq.store.DispatchRequest; +import com.alibaba.rocketmq.store.config.StorePathConfigHelper; + + +/** + * 消息索引服务 + * + * @author shijia.wxr + * @since 2013-7-21 + */ +public class IndexService extends ServiceThread { + private static final Logger log = LoggerFactory.getLogger(LoggerName.StoreLoggerName); + private final DefaultMessageStore defaultMessageStore; + // 索引配置 + private final int hashSlotNum; + private final int indexNum; + private final String storePath; + // 索引文件集合 + private final ArrayList indexFileList = new ArrayList(); + // 读写锁（针对indexFileList） + private final ReadWriteLock readWriteLock = new ReentrantReadWriteLock(); + private LinkedBlockingQueue requestQueue = new LinkedBlockingQueue(300000); + + + public IndexService(final DefaultMessageStore store) { + this.defaultMessageStore = store; + this.hashSlotNum = store.getMessageStoreConfig().getMaxHashSlotNum(); + this.indexNum = store.getMessageStoreConfig().getMaxIndexNum(); + this.storePath = + StorePathConfigHelper.getStorePathIndex(store.getMessageStoreConfig().getStorePathRootDir()); + } + + + public boolean load(final boolean lastExitOK) { + File dir = new File(this.storePath); + File[] files = dir.listFiles(); + if (files != null) { + // ascending order + Arrays.sort(files); + for (File file : files) { + try { + IndexFile f = new IndexFile(file.getPath(), this.hashSlotNum, this.indexNum, 0, 0); + f.load(); + + if (!lastExitOK) { + if (f.getEndTimestamp() > this.defaultMessageStore.getStoreCheckpoint() + .getIndexMsgTimestamp()) { + f.destroy(0); + continue; + } + } + + log.info("load index file OK, " + f.getFileName()); + this.indexFileList.add(f); + } + catch (IOException e) { + log.error("load file " + file + " error", e); + return false; + } + } + } + + return true; + } + + + /** + * 删除索引文件 + */ + public void deleteExpiredFile(long offset) { + Object[] files = null; + try { + this.readWriteLock.readLock().lock(); + if (this.indexFileList.isEmpty()) { + return; + } + + long endPhyOffset = this.indexFileList.get(0).getEndPhyOffset(); + if (endPhyOffset < offset) { + files = this.indexFileList.toArray(); + } + } + catch (Exception e) { + log.error("destroy exception", e); + } + finally { + this.readWriteLock.readLock().unlock(); + } + + if (files != null) { + List fileList = new ArrayList(); + for (int i = 0; i < (files.length - 1); i++) { + IndexFile f = (IndexFile) files[i]; + if (f.getEndPhyOffset() < offset) { + fileList.add(f); + } + else { + break; + } + } + + this.deleteExpiredFile(fileList); + } + } + + + /** + * 删除文件只能从头开始删 + */ + private void deleteExpiredFile(List files) { + if (!files.isEmpty()) { + try { + this.readWriteLock.writeLock().lock(); + for (IndexFile file : files) { + boolean destroyed = file.destroy(3000); + destroyed = destroyed && this.indexFileList.remove(file); + if (!destroyed) { + log.error("deleteExpiredFile remove failed."); + break; + } + } + } + catch (Exception e) { + log.error("deleteExpiredFile has exception.", e); + } + finally { + this.readWriteLock.writeLock().unlock(); + } + } + } + + + public void destroy() { + try { + this.readWriteLock.readLock().lock(); + for (IndexFile f : this.indexFileList) { + f.destroy(1000 * 3); + } + this.indexFileList.clear(); + } + catch (Exception e) { + log.error("destroy exception", e); + } + finally { + this.readWriteLock.readLock().unlock(); + } + } + + + public QueryOffsetResult queryOffset(String topic, String key, int maxNum, long begin, long end) { + List phyOffsets = new ArrayList(maxNum); + // TODO 可能需要返回给最终用户 + long indexLastUpdateTimestamp = 0; + long indexLastUpdatePhyoffset = 0; + maxNum = Math.min(maxNum, this.defaultMessageStore.getMessageStoreConfig().getMaxMsgsNumBatch()); + try { + this.readWriteLock.readLock().lock(); + if (!this.indexFileList.isEmpty()) { + for (int i = this.indexFileList.size(); i > 0; i--) { + IndexFile f = this.indexFileList.get(i - 1); + boolean lastFile = (i == this.indexFileList.size()); + if (lastFile) { + indexLastUpdateTimestamp = f.getEndTimestamp(); + indexLastUpdatePhyoffset = f.getEndPhyOffset(); + } + + if (f.isTimeMatched(begin, end)) { + // 最后一个文件需要加锁 + f.selectPhyOffset(phyOffsets, this.buildKey(topic, key), maxNum, begin, end, lastFile); + } + + // 再往前遍历时间更不符合 + if (f.getBeginTimestamp() < begin) { + break; + } + + if (phyOffsets.size() >= maxNum) { + break; + } + } + } + } + catch (Exception e) { + log.error("queryMsg exception", e); + } + finally { + this.readWriteLock.readLock().unlock(); + } + + return new QueryOffsetResult(phyOffsets, indexLastUpdateTimestamp, indexLastUpdatePhyoffset); + } + + + private String buildKey(final String topic, final String key) { + return topic + "#" + key; + } + + + /** + * 向队列中添加请求，队列满情况下，丢弃请求 + */ + public void putRequest(final Object[] reqs) { + boolean offer = this.requestQueue.offer(reqs); + if (!offer) { + if (log.isDebugEnabled()) { + log.debug("putRequest index failed, {}", reqs); + } + } + } + + + @Override + public void run() { + log.info(this.getServiceName() + " service started"); + + while (!this.isStoped()) { + try { + Object[] req = this.requestQueue.poll(3000, TimeUnit.MILLISECONDS); + + if (req != null) { + this.buildIndex(req); + } + } + catch (Exception e) { + log.warn(this.getServiceName() + " service has exception. ", e); + } + } + + log.info(this.getServiceName() + " service end"); + } + + + public void buildIndex(Object[] req) { + boolean breakdown = false; + IndexFile indexFile = retryGetAndCreateIndexFile(); + if (indexFile != null) { + long endPhyOffset = indexFile.getEndPhyOffset(); + MSG_WHILE: for (Object o : req) { + DispatchRequest msg = (DispatchRequest) o; + String topic = msg.getTopic(); + String keys = msg.getKeys(); + if (msg.getCommitLogOffset() < endPhyOffset) { + continue; + } + + final int tranType = MessageSysFlag.getTransactionValue(msg.getSysFlag()); + switch (tranType) { + case MessageSysFlag.TransactionNotType: + case MessageSysFlag.TransactionPreparedType: + break; + case MessageSysFlag.TransactionCommitType: + case MessageSysFlag.TransactionRollbackType: + continue; + } + + if (keys != null && keys.length() > 0) { + String[] keyset = keys.split(MessageConst.KEY_SEPARATOR); + for (String key : keyset) { + // TODO 是否需要TRIM + if (key.length() > 0) { + for (boolean ok = + indexFile.putKey(buildKey(topic, key), msg.getCommitLogOffset(), + msg.getStoreTimestamp()); !ok;) { + log.warn("index file full, so create another one, " + indexFile.getFileName()); + indexFile = retryGetAndCreateIndexFile(); + if (null == indexFile) { + breakdown = true; + break MSG_WHILE; + } + + ok = + indexFile.putKey(buildKey(topic, key), msg.getCommitLogOffset(), + msg.getStoreTimestamp()); + } + } + } + } + } + } + // IO发生故障，build索引过程中断，需要人工参与处理 + else { + breakdown = true; + } + + if (breakdown) { + log.error("build index error, stop building index"); + } + } + + + public IndexFile retryGetAndCreateIndexFile() { + IndexFile indexFile = null; + + // 如果创建失败，尝试重建3次 + for (int times = 0; null == indexFile && times < 3; times++) { + indexFile = this.getAndCreateLastIndexFile(); + if (null != indexFile) + break; + + try { + log.error("try to create index file, " + times + " times"); + Thread.sleep(1000); + } + catch (InterruptedException e) { + e.printStackTrace(); + } + } + + // 重试多次，仍然无法创建索引文件 + if (null == indexFile) { + this.defaultMessageStore.getAccessRights().makeIndexFileError(); + log.error("mark index file can not build flag"); + } + + return indexFile; + } + + + /** + * 获取最后一个索引文件，如果集合为空或者最后一个文件写满了，则新建一个文件
+ * 只有一个线程调用，所以不存在写竟争问题 + */ + public IndexFile getAndCreateLastIndexFile() { + IndexFile indexFile = null; + IndexFile prevIndexFile = null; + long lastUpdateEndPhyOffset = 0; + long lastUpdateIndexTimestamp = 0; + // 先尝试使用读锁 + { + this.readWriteLock.readLock().lock(); + if (!this.indexFileList.isEmpty()) { + IndexFile tmp = this.indexFileList.get(this.indexFileList.size() - 1); + if (!tmp.isWriteFull()) { + indexFile = tmp; + } + else { + lastUpdateEndPhyOffset = tmp.getEndPhyOffset(); + lastUpdateIndexTimestamp = tmp.getEndTimestamp(); + prevIndexFile = tmp; + } + } + + this.readWriteLock.readLock().unlock(); + } + + // 如果没找到，使用写锁创建文件 + if (indexFile == null) { + try { + String fileName = + this.storePath + File.separator + + UtilAll.timeMillisToHumanString(System.currentTimeMillis()); + indexFile = + new IndexFile(fileName, this.hashSlotNum, this.indexNum, lastUpdateEndPhyOffset, + lastUpdateIndexTimestamp); + this.readWriteLock.writeLock().lock(); + this.indexFileList.add(indexFile); + } + catch (Exception e) { + log.error("getLastIndexFile exception ", e); + } + finally { + this.readWriteLock.writeLock().unlock(); + } + + // 每创建一个新文件，之前文件要刷盘 + if (indexFile != null) { + final IndexFile flushThisFile = prevIndexFile; + Thread flushThread = new Thread(new Runnable() { + @Override + public void run() { + IndexService.this.flush(flushThisFile); + } + }, "FlushIndexFileThread"); + + flushThread.setDaemon(true); + flushThread.start(); + } + } + + return indexFile; + } + + + public void flush(final IndexFile f) { + if (null == f) + return; + + long indexMsgTimestamp = 0; + + if (f.isWriteFull()) { + indexMsgTimestamp = f.getEndTimestamp(); + } + + f.flush(); + + if (indexMsgTimestamp > 0) { + this.defaultMessageStore.getStoreCheckpoint().setIndexMsgTimestamp(indexMsgTimestamp); + this.defaultMessageStore.getStoreCheckpoint().flush(); + } + } + + + @Override + public String getServiceName() { + return IndexService.class.getSimpleName(); + } +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/index/QueryOffsetResult.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/index/QueryOffsetResult.java new file mode 100644 index 000000000..30246ebe5 --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/index/QueryOffsetResult.java @@ -0,0 +1,54 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store.index; + +import java.util.List; + + +/** + * 根据索引查询消息，返回结果 + * + * @author shijia.wxr + * @since 2013-7-21 + */ +public class QueryOffsetResult { + private final List phyOffsets; + private final long indexLastUpdateTimestamp; + private final long indexLastUpdatePhyoffset; + + + public QueryOffsetResult(List phyOffsets, long indexLastUpdateTimestamp, + long indexLastUpdatePhyoffset) { + this.phyOffsets = phyOffsets; + this.indexLastUpdateTimestamp = indexLastUpdateTimestamp; + this.indexLastUpdatePhyoffset = indexLastUpdatePhyoffset; + } + + + public List getPhyOffsets() { + return phyOffsets; + } + + + public long getIndexLastUpdateTimestamp() { + return indexLastUpdateTimestamp; + } + + + public long getIndexLastUpdatePhyoffset() { + return indexLastUpdatePhyoffset; + } +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/schedule/DelayOffsetSerializeWrapper.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/schedule/DelayOffsetSerializeWrapper.java new file mode 100644 index 000000000..3835648c0 --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/schedule/DelayOffsetSerializeWrapper.java @@ -0,0 +1,42 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store.schedule; + +import java.util.concurrent.ConcurrentHashMap; + +import com.alibaba.rocketmq.remoting.protocol.RemotingSerializable; + + +/** + * 延时消息进度，序列化包装 + * + * @author shijia.wxr + * @since 2013-7-21 + */ +public class DelayOffsetSerializeWrapper extends RemotingSerializable { + private ConcurrentHashMap offsetTable = + new ConcurrentHashMap(32); + + + public ConcurrentHashMap getOffsetTable() { + return offsetTable; + } + + + public void setOffsetTable(ConcurrentHashMap offsetTable) { + this.offsetTable = offsetTable; + } +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/schedule/ScheduleMessageService.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/schedule/ScheduleMessageService.java new file mode 100644 index 000000000..ad8c206f8 --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/schedule/ScheduleMessageService.java @@ -0,0 +1,411 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store.schedule; + +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map.Entry; +import java.util.Timer; +import java.util.TimerTask; +import java.util.concurrent.ConcurrentHashMap; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.rocketmq.common.ConfigManager; +import com.alibaba.rocketmq.common.TopicFilterType; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.common.message.MessageAccessor; +import com.alibaba.rocketmq.common.message.MessageConst; +import com.alibaba.rocketmq.common.message.MessageDecoder; +import com.alibaba.rocketmq.common.message.MessageExt; +import com.alibaba.rocketmq.common.running.RunningStats; +import com.alibaba.rocketmq.store.ConsumeQueue; +import com.alibaba.rocketmq.store.DefaultMessageStore; +import com.alibaba.rocketmq.store.MessageExtBrokerInner; +import com.alibaba.rocketmq.store.PutMessageResult; +import com.alibaba.rocketmq.store.PutMessageStatus; +import com.alibaba.rocketmq.store.SelectMapedBufferResult; +import com.alibaba.rocketmq.store.config.StorePathConfigHelper; + + +/** + * 定时消息服务 + * + * @author shijia.wxr + * @since 2013-7-21 + */ +public class ScheduleMessageService extends ConfigManager { + public static final String SCHEDULE_TOPIC = "SCHEDULE_TOPIC_XXXX"; + private static final Logger log = LoggerFactory.getLogger(LoggerName.StoreLoggerName); + private static final long FIRST_DELAY_TIME = 1000L; + private static final long DELAY_FOR_A_WHILE = 100L; + private static final long DELAY_FOR_A_PERIOD = 10000L; + // 每个level对应的延时时间 + private final ConcurrentHashMap delayLevelTable = + new ConcurrentHashMap(32); + // 延时计算到了哪里 + private final ConcurrentHashMap offsetTable = + new ConcurrentHashMap(32); + // 定时器 + private final Timer timer = new Timer("ScheduleMessageTimerThread", true); + // 存储顶层对象 + private final DefaultMessageStore defaultMessageStore; + // 最大值 + private int maxDelayLevel; + + + public ScheduleMessageService(final DefaultMessageStore defaultMessageStore) { + this.defaultMessageStore = defaultMessageStore; + } + + + public void buildRunningStats(HashMap stats) { + Iterator> it = this.offsetTable.entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + int queueId = delayLevel2QueueId(next.getKey()); + long delayOffset = next.getValue(); + long maxOffset = this.defaultMessageStore.getMaxOffsetInQuque(SCHEDULE_TOPIC, queueId); + String value = String.format("%d,%d", delayOffset, maxOffset); + String key = String.format("%s_%d", RunningStats.scheduleMessageOffset.name(), next.getKey()); + stats.put(key, value); + } + } + + + public static int queueId2DelayLevel(final int queueId) { + return queueId + 1; + } + + + public static int delayLevel2QueueId(final int delayLevel) { + return delayLevel - 1; + } + + + private void updateOffset(int delayLevel, long offset) { + this.offsetTable.put(delayLevel, offset); + } + + + public long computeDeliverTimestamp(final int delayLevel, final long storeTimestamp) { + Long time = this.delayLevelTable.get(delayLevel); + if (time != null) { + return time + storeTimestamp; + } + + return storeTimestamp + 1000; + } + + + public void start() { + // 为每个延时队列增加定时器 + for (Integer level : this.delayLevelTable.keySet()) { + Long timeDelay = this.delayLevelTable.get(level); + Long offset = this.offsetTable.get(level); + if (null == offset) { + offset = 0L; + } + + if (timeDelay != null) { + this.timer.schedule(new DeliverDelayedMessageTimerTask(level, offset), FIRST_DELAY_TIME); + } + } + + // 定时将延时进度刷盘 + this.timer.scheduleAtFixedRate(new TimerTask() { + + @Override + public void run() { + try { + ScheduleMessageService.this.persist(); + } + catch (Exception e) { + log.error("scheduleAtFixedRate flush exception", e); + } + } + }, 10000, this.defaultMessageStore.getMessageStoreConfig().getFlushDelayOffsetInterval()); + } + + + public void shutdown() { + this.timer.cancel(); + } + + + public int getMaxDelayLevel() { + return maxDelayLevel; + } + + + public String encode() { + return this.encode(false); + } + + + public String encode(final boolean prettyFormat) { + DelayOffsetSerializeWrapper delayOffsetSerializeWrapper = new DelayOffsetSerializeWrapper(); + delayOffsetSerializeWrapper.setOffsetTable(this.offsetTable); + return delayOffsetSerializeWrapper.toJson(prettyFormat); + } + + + @Override + public void decode(String jsonString) { + if (jsonString != null) { + DelayOffsetSerializeWrapper delayOffsetSerializeWrapper = + DelayOffsetSerializeWrapper.fromJson(jsonString, DelayOffsetSerializeWrapper.class); + if (delayOffsetSerializeWrapper != null) { + this.offsetTable.putAll(delayOffsetSerializeWrapper.getOffsetTable()); + } + } + } + + + @Override + public String configFilePath() { + return StorePathConfigHelper.getDelayOffsetStorePath(this.defaultMessageStore.getMessageStoreConfig() + .getStorePathRootDir()); + } + + + public boolean load() { + boolean result = super.load(); + result = result && this.parseDelayLevel(); + return result; + } + + + public boolean parseDelayLevel() { + HashMap timeUnitTable = new HashMap(); + timeUnitTable.put("s", 1000L); + timeUnitTable.put("m", 1000L * 60); + timeUnitTable.put("h", 1000L * 60 * 60); + timeUnitTable.put("d", 1000L * 60 * 60 * 24); + + String levelString = this.defaultMessageStore.getMessageStoreConfig().getMessageDelayLevel(); + try { + String[] levelArray = levelString.split(" "); + for (int i = 0; i < levelArray.length; i++) { + String value = levelArray[i]; + String ch = value.substring(value.length() - 1); + Long tu = timeUnitTable.get(ch); + + int level = i + 1; + if (level > this.maxDelayLevel) { + this.maxDelayLevel = level; + } + long num = Long.parseLong(value.substring(0, value.length() - 1)); + long delayTimeMillis = tu * num; + this.delayLevelTable.put(level, delayTimeMillis); + } + } + catch (Exception e) { + log.error("parseDelayLevel exception", e); + log.info("levelString String = {}", levelString); + return false; + } + + return true; + } + + class DeliverDelayedMessageTimerTask extends TimerTask { + private final int delayLevel; + private final long offset; + + + public DeliverDelayedMessageTimerTask(int delayLevel, long offset) { + this.delayLevel = delayLevel; + this.offset = offset; + } + + + @Override + public void run() { + try { + this.executeOnTimeup(); + } + catch (Exception e) { + // XXX: warn and notify me + log.error("ScheduleMessageService, executeOnTimeup exception", e); + ScheduleMessageService.this.timer.schedule(new DeliverDelayedMessageTimerTask( + this.delayLevel, this.offset), DELAY_FOR_A_PERIOD); + } + } + + + /** + * 纠正下次投递时间，如果时间特别大，则纠正为当前时间 + * + * @return + */ + private long correctDeliverTimestamp(final long now, final long deliverTimestamp) { + // 如果为0，则会立刻投递 + long result = deliverTimestamp; + // 超过最大值，纠正为当前时间 + long maxTimestamp = now + ScheduleMessageService.this.delayLevelTable.get(this.delayLevel); + if (deliverTimestamp > maxTimestamp) { + result = now; + } + + return result; + } + + + public void executeOnTimeup() { + ConsumeQueue cq = + ScheduleMessageService.this.defaultMessageStore.findConsumeQueue(SCHEDULE_TOPIC, + delayLevel2QueueId(delayLevel)); + + long failScheduleOffset = offset; + + if (cq != null) { + SelectMapedBufferResult bufferCQ = cq.getIndexBuffer(this.offset); + if (bufferCQ != null) { + try { + long nextOffset = offset; + int i = 0; + for (; i < bufferCQ.getSize(); i += ConsumeQueue.CQStoreUnitSize) { + long offsetPy = bufferCQ.getByteBuffer().getLong(); + int sizePy = bufferCQ.getByteBuffer().getInt(); + long tagsCode = bufferCQ.getByteBuffer().getLong(); + + // 队列里存储的tagsCode实际是一个时间点 + long now = System.currentTimeMillis(); + long deliverTimestamp = this.correctDeliverTimestamp(now, tagsCode); + + nextOffset = offset + (i / ConsumeQueue.CQStoreUnitSize); + + long countdown = deliverTimestamp - now; + // 时间到了，该投递 + if (countdown <= 0) { + MessageExt msgExt = + ScheduleMessageService.this.defaultMessageStore.lookMessageByOffset( + offsetPy, sizePy); + + if (msgExt != null) { + try { + MessageExtBrokerInner msgInner = this.messageTimeup(msgExt); + PutMessageResult putMessageResult = + ScheduleMessageService.this.defaultMessageStore + .putMessage(msgInner); + // 成功 + if (putMessageResult != null + && putMessageResult.getPutMessageStatus() == PutMessageStatus.PUT_OK) { + continue; + } + // 失败 + else { + // XXX: warn and notify me + log.error( + "ScheduleMessageService, a message time up, but reput it failed, topic: {} msgId {}", + msgExt.getTopic(), msgExt.getMsgId()); + ScheduleMessageService.this.timer.schedule( + new DeliverDelayedMessageTimerTask(this.delayLevel, + nextOffset), DELAY_FOR_A_PERIOD); + ScheduleMessageService.this.updateOffset(this.delayLevel, + nextOffset); + return; + } + } + catch (Exception e) { + /* + * XXX: warn and notify me + * msgExt里面的内容不完整 + * ，如没有REAL_QID,REAL_TOPIC之类的 + * ，导致数据无法正常的投递到正确的消费队列，所以暂时先直接跳过该条消息 + */ + log.error( + "ScheduleMessageService, messageTimeup execute error, drop it. msgExt=" + + msgExt + ", nextOffset=" + nextOffset + ",offsetPy=" + + offsetPy + ",sizePy=" + sizePy, e); + } + } + } + // 时候未到，继续定时 + else { + ScheduleMessageService.this.timer.schedule( + new DeliverDelayedMessageTimerTask(this.delayLevel, nextOffset), + countdown); + ScheduleMessageService.this.updateOffset(this.delayLevel, nextOffset); + return; + } + } // end of for + + nextOffset = offset + (i / ConsumeQueue.CQStoreUnitSize); + ScheduleMessageService.this.timer.schedule(new DeliverDelayedMessageTimerTask( + this.delayLevel, nextOffset), DELAY_FOR_A_WHILE); + ScheduleMessageService.this.updateOffset(this.delayLevel, nextOffset); + return; + } + finally { + // 必须释放资源 + bufferCQ.release(); + } + } // end of if (bufferCQ != null) + else { + /* + * 索引文件被删除，定时任务中记录的offset已经被删除，会导致从该位置中取不到数据， + * 这里直接纠正下一次定时任务的offset为当前定时任务队列的最小值 + */ + long cqMinOffset = cq.getMinOffsetInQuque(); + if (offset < cqMinOffset) { + failScheduleOffset = cqMinOffset; + log.error("schedule CQ offset invalid. offset=" + offset + ", cqMinOffset=" + + cqMinOffset + ", queueId=" + cq.getQueueId()); + } + } + } // end of if (cq != null) + + ScheduleMessageService.this.timer.schedule(new DeliverDelayedMessageTimerTask(this.delayLevel, + failScheduleOffset), DELAY_FOR_A_WHILE); + } + + + private MessageExtBrokerInner messageTimeup(MessageExt msgExt) { + MessageExtBrokerInner msgInner = new MessageExtBrokerInner(); + msgInner.setBody(msgExt.getBody()); + msgInner.setFlag(msgExt.getFlag()); + MessageAccessor.setProperties(msgInner, msgExt.getProperties()); + + TopicFilterType topicFilterType = MessageExt.parseTopicFilterType(msgInner.getSysFlag()); + long tagsCodeValue = + MessageExtBrokerInner.tagsString2tagsCode(topicFilterType, msgInner.getTags()); + msgInner.setTagsCode(tagsCodeValue); + msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgExt.getProperties())); + + msgInner.setSysFlag(msgExt.getSysFlag()); + msgInner.setBornTimestamp(msgExt.getBornTimestamp()); + msgInner.setBornHost(msgExt.getBornHost()); + msgInner.setStoreHost(msgExt.getStoreHost()); + msgInner.setReconsumeTimes(msgExt.getReconsumeTimes()); + + msgInner.setWaitStoreMsgOK(false); + MessageAccessor.clearProperty(msgInner, MessageConst.PROPERTY_DELAY_TIME_LEVEL); + + // 恢复Topic + msgInner.setTopic(msgInner.getProperty(MessageConst.PROPERTY_REAL_TOPIC)); + + // 恢复QueueId + String queueIdStr = msgInner.getProperty(MessageConst.PROPERTY_REAL_QUEUE_ID); + int queueId = Integer.parseInt(queueIdStr); + msgInner.setQueueId(queueId); + + return msgInner; + } + } +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/stats/BrokerStats.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/stats/BrokerStats.java new file mode 100644 index 000000000..8f31cc1c9 --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/stats/BrokerStats.java @@ -0,0 +1,117 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.store.stats; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.store.DefaultMessageStore; + + +/** + * Broker上的一些统计数据 + * + * @author shijia.wxr + * @since 2013-10-23 + */ +public class BrokerStats { + private static final Logger log = LoggerFactory.getLogger(LoggerName.BrokerLoggerName); + + // 昨天凌晨00:00:00记录的put消息总数 + private volatile long msgPutTotalYesterdayMorning; + // 今天凌晨00:00:00记录的put消息总数 + private volatile long msgPutTotalTodayMorning; + + // 昨天凌晨00:00:00记录的get消息总数 + private volatile long msgGetTotalYesterdayMorning; + // 今天凌晨00:00:00记录的get消息总数 + private volatile long msgGetTotalTodayMorning; + + private final DefaultMessageStore defaultMessageStore; + + + public BrokerStats(DefaultMessageStore defaultMessageStore) { + this.defaultMessageStore = defaultMessageStore; + } + + + /** + * 每天00:00:00调用 + */ + public void record() { + this.msgPutTotalYesterdayMorning = this.msgPutTotalTodayMorning; + this.msgGetTotalYesterdayMorning = this.msgGetTotalTodayMorning; + + this.msgPutTotalTodayMorning = + this.defaultMessageStore.getStoreStatsService().getPutMessageTimesTotal(); + this.msgGetTotalTodayMorning = + this.defaultMessageStore.getStoreStatsService().getGetMessageTransferedMsgCount().get(); + + log.info("yesterday put message total: {}", msgPutTotalTodayMorning - msgPutTotalYesterdayMorning); + log.info("yesterday get message total: {}", msgGetTotalTodayMorning - msgGetTotalYesterdayMorning); + } + + + public long getMsgPutTotalYesterdayMorning() { + return msgPutTotalYesterdayMorning; + } + + + public void setMsgPutTotalYesterdayMorning(long msgPutTotalYesterdayMorning) { + this.msgPutTotalYesterdayMorning = msgPutTotalYesterdayMorning; + } + + + public long getMsgPutTotalTodayMorning() { + return msgPutTotalTodayMorning; + } + + + public void setMsgPutTotalTodayMorning(long msgPutTotalTodayMorning) { + this.msgPutTotalTodayMorning = msgPutTotalTodayMorning; + } + + + public long getMsgGetTotalYesterdayMorning() { + return msgGetTotalYesterdayMorning; + } + + + public void setMsgGetTotalYesterdayMorning(long msgGetTotalYesterdayMorning) { + this.msgGetTotalYesterdayMorning = msgGetTotalYesterdayMorning; + } + + + public long getMsgGetTotalTodayMorning() { + return msgGetTotalTodayMorning; + } + + + public void setMsgGetTotalTodayMorning(long msgGetTotalTodayMorning) { + this.msgGetTotalTodayMorning = msgGetTotalTodayMorning; + } + + + public long getMsgPutTotalTodayNow() { + return this.defaultMessageStore.getStoreStatsService().getPutMessageTimesTotal(); + } + + + public long getMsgGetTotalTodayNow() { + return this.defaultMessageStore.getStoreStatsService().getGetMessageTransferedMsgCount().get(); + } +} diff --git a/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/stats/BrokerStatsManager.java b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/stats/BrokerStatsManager.java new file mode 100644 index 000000000..3490a95e5 --- /dev/null +++ b/rocketmq-store/src/main/java/com/alibaba/rocketmq/store/stats/BrokerStatsManager.java @@ -0,0 +1,148 @@ +package com.alibaba.rocketmq.store.stats; + +import com.alibaba.rocketmq.common.ThreadFactoryImpl; +import com.alibaba.rocketmq.common.constant.LoggerName; +import com.alibaba.rocketmq.common.stats.StatsItem; +import com.alibaba.rocketmq.common.stats.StatsItemSet; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; + + +public class BrokerStatsManager { + private static final Logger log = LoggerFactory.getLogger(LoggerName.RocketmqStatsLoggerName); + private final ScheduledExecutorService scheduledExecutorService = Executors + .newSingleThreadScheduledExecutor(new ThreadFactoryImpl("BrokerStatsThread")); + + public static final String TOPIC_PUT_NUMS = "TOPIC_PUT_NUMS"; + public static final String TOPIC_PUT_SIZE = "TOPIC_PUT_SIZE"; + public static final String GROUP_GET_NUMS = "GROUP_GET_NUMS"; + public static final String GROUP_GET_SIZE = "GROUP_GET_SIZE"; + public static final String GROUP_GET_FROM_DISK_NUMS = "GROUP_GET_FROM_DISK_NUMS"; + public static final String GROUP_GET_FROM_DISK_SIZE = "GROUP_GET_FROM_DISK_SIZE"; + public static final String SNDBCK_PUT_NUMS = "SNDBCK_PUT_NUMS"; + public static final String BROKER_PUT_NUMS = "BROKER_PUT_NUMS"; + public static final String BROKER_GET_NUMS = "BROKER_GET_NUMS"; + public static final String BROKER_GET_FROM_DISK_NUMS = "BROKER_GET_FROM_DISK_NUMS"; + + // Topic Put Nums + private final StatsItemSet topicPutNums = new StatsItemSet(TOPIC_PUT_NUMS, this.scheduledExecutorService, + log); + + // Topic Put Size + private final StatsItemSet topicPutSize = new StatsItemSet(TOPIC_PUT_SIZE, this.scheduledExecutorService, + log); + + // Topic@ConsumerGroup Get Nums + private final StatsItemSet groupGetNums = new StatsItemSet(GROUP_GET_NUMS, this.scheduledExecutorService, + log); + + // Topic@ConsumerGroup Get Size + private final StatsItemSet groupGetSize = new StatsItemSet(GROUP_GET_SIZE, this.scheduledExecutorService, + log); + + // Topic@ConsumerGroup get in disk nums + private final StatsItemSet groupGetFromDiskNums = new StatsItemSet(GROUP_GET_FROM_DISK_NUMS, + this.scheduledExecutorService, log); + + // Topic@ConsumerGroup group get in disk size + private final StatsItemSet groupGetFromDiskSize = new StatsItemSet(GROUP_GET_FROM_DISK_SIZE, + this.scheduledExecutorService, log); + + // Broker Put Nums + private final StatsItem brokerPutNums; + + // Broker Get Nums + private final StatsItem brokerGetNums; + + // Broker Get From Disk Nums + private final StatsItem brokerGetFromDiskNums; + + // Topic@ConsumerGroup sendback Nums + private final StatsItemSet sndbckPutNums = new StatsItemSet(SNDBCK_PUT_NUMS, + this.scheduledExecutorService, log); + + + public BrokerStatsManager(String clusterName) { + // Broker Put Nums + this.brokerPutNums = new StatsItem(BROKER_PUT_NUMS, // + clusterName, this.scheduledExecutorService, log); + + // Broker Get Nums + this.brokerGetNums = new StatsItem(BROKER_GET_NUMS, // + clusterName, this.scheduledExecutorService, log); + + // Broker Get From Disk Nums + this.brokerGetFromDiskNums = new StatsItem(BROKER_GET_FROM_DISK_NUMS, // + clusterName, this.scheduledExecutorService, log); + } + + + public void start() { + this.brokerPutNums.init(); + this.brokerGetNums.init(); + this.brokerGetFromDiskNums.init(); + } + + + public void shutdown() { + this.scheduledExecutorService.shutdown(); + } + + + public void incTopicPutNums(final String topic) { + this.topicPutNums.addValue(topic, 1, 1); + } + + + public void incTopicPutSize(final String topic, final int size) { + this.topicPutSize.addValue(topic, size, 1); + } + + + public void incGroupGetNums(final String group, final String topic, final int incValue) { + this.groupGetNums.addValue(topic + "@" + group, incValue, 1); + } + + + public void incGroupGetSize(final String group, final String topic, final int incValue) { + this.groupGetSize.addValue(topic + "@" + group, incValue, 1); + } + + + public void incBrokerPutNums() { + this.brokerPutNums.getValue().incrementAndGet(); + } + + + public void incBrokerGetNums(final int incValue) { + this.brokerGetNums.getValue().addAndGet(incValue); + } + + + public void incSendBackNums(final String group, final String topic) { + this.sndbckPutNums.addValue(topic + "@" + group, 1, 1); + } + + + public double tpsGroupGetNums(final String group, final String topic) { + return this.groupGetNums.getStatsDataInMinute(topic + "@" + group).getTps(); + } + + + public void incBrokerGetFromDiskNums(final int incValue) { + this.brokerGetFromDiskNums.getValue().addAndGet(incValue); + } + + + public void incGroupGetFromDiskNums(final String group, final String topic, final int incValue) { + this.groupGetFromDiskNums.addValue(topic + "@" + group, incValue, 1); + } + + + public void incGroupGetFromDiskSize(final String group, final String topic, final int incValue) { + this.groupGetFromDiskSize.addValue(topic + "@" + group, incValue, 1); + } +} diff --git a/rocketmq-store/src/test/java/com/alibaba/rocketmq/store/DefaultMessageStoreTest.java b/rocketmq-store/src/test/java/com/alibaba/rocketmq/store/DefaultMessageStoreTest.java new file mode 100644 index 000000000..1a1ac8a5e --- /dev/null +++ b/rocketmq-store/src/test/java/com/alibaba/rocketmq/store/DefaultMessageStoreTest.java @@ -0,0 +1,173 @@ +package com.alibaba.rocketmq.store; + +import com.alibaba.rocketmq.store.config.FlushDiskType; +import com.alibaba.rocketmq.store.config.MessageStoreConfig; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.Assert.assertTrue; + + +/** + * @author shijia.wxr + */ +public class DefaultMessageStoreTest { + // 队列个数 + private static int QUEUE_TOTAL = 100; + // 发往哪个队列 + private static AtomicInteger QueueId = new AtomicInteger(0); + // 发送主机地址 + private static SocketAddress BornHost; + // 存储主机地址 + private static SocketAddress StoreHost; + // 消息体 + private static byte[] MessageBody; + + private static final String StoreMessage = "Once, there was a chance for me!"; + + + public MessageExtBrokerInner buildMessage() { + MessageExtBrokerInner msg = new MessageExtBrokerInner(); + msg.setTopic("AAA"); + msg.setTags("TAG1"); + msg.setKeys("Hello"); + msg.setBody(MessageBody); + msg.setKeys(String.valueOf(System.currentTimeMillis())); + msg.setQueueId(Math.abs(QueueId.getAndIncrement()) % QUEUE_TOTAL); + msg.setSysFlag(4); + msg.setBornTimestamp(System.currentTimeMillis()); + msg.setStoreHost(StoreHost); + msg.setBornHost(BornHost); + + return msg; + } + + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + StoreHost = new InetSocketAddress(InetAddress.getLocalHost(), 8123); + BornHost = new InetSocketAddress(InetAddress.getByName("127.0.0.1"), 0); + + } + + + @AfterClass + public static void tearDownAfterClass() throws Exception { + } + + + @Test + public void test_write_read() throws Exception { + System.out.println("================================================================"); + long totalMsgs = 10000; + QUEUE_TOTAL = 1; + + // 构造消息体 + MessageBody = StoreMessage.getBytes(); + + MessageStoreConfig messageStoreConfig = new MessageStoreConfig(); + // 每个物理映射文件 4K + messageStoreConfig.setMapedFileSizeCommitLog(1024 * 8); + messageStoreConfig.setMapedFileSizeConsumeQueue(1024 * 4); + messageStoreConfig.setMaxHashSlotNum(100); + messageStoreConfig.setMaxIndexNum(100 * 10); + + MessageStore master = new DefaultMessageStore(messageStoreConfig, null); + // 第一步，load已有数据 + boolean load = master.load(); + assertTrue(load); + + // 第二步，启动服务 + master.start(); + for (long i = 0; i < totalMsgs; i++) { + PutMessageResult result = master.putMessage(buildMessage()); + + System.out.println(i + "\t" + result.getAppendMessageResult().getMsgId()); + } + + // 开始读文件 + for (long i = 0; i < totalMsgs; i++) { + try { + GetMessageResult result = master.getMessage("GROUP_A", "TOPIC_A", 0, i, 1024 * 1024, null); + if (result == null) { + System.out.println("result == null " + i); + } + assertTrue(result != null); + result.release(); + System.out.println("read " + i + " OK"); + } + catch (Exception e) { + e.printStackTrace(); + } + + } + + // 关闭存储服务 + master.shutdown(); + + // 删除文件 + master.destroy(); + System.out.println("================================================================"); + } + + + @Test + public void test_group_commit() throws Exception { + System.out.println("================================================================"); + long totalMsgs = 10000; + QUEUE_TOTAL = 1; + + // 构造消息体 + MessageBody = StoreMessage.getBytes(); + + MessageStoreConfig messageStoreConfig = new MessageStoreConfig(); + // 每个物理映射文件 4K + messageStoreConfig.setMapedFileSizeCommitLog(1024 * 8); + + // 开启GroupCommit功能 + messageStoreConfig.setFlushDiskType(FlushDiskType.SYNC_FLUSH); + + MessageStore master = new DefaultMessageStore(messageStoreConfig, null); + // 第一步，load已有数据 + boolean load = master.load(); + assertTrue(load); + + // 第二步，启动服务 + master.start(); + for (long i = 0; i < totalMsgs; i++) { + PutMessageResult result = master.putMessage(buildMessage()); + + System.out.println(i + "\t" + result.getAppendMessageResult().getMsgId()); + } + + // 开始读文件 + for (long i = 0; i < totalMsgs; i++) { + try { + GetMessageResult result = master.getMessage("GROUP_A", "TOPIC_A", 0, i, 1024 * 1024, null); + if (result == null) { + System.out.println("result == null " + i); + } + assertTrue(result != null); + result.release(); + System.out.println("read " + i + " OK"); + } + catch (Exception e) { + e.printStackTrace(); + } + + } + + // 关闭存储服务 + master.shutdown(); + + // 删除文件 + master.destroy(); + System.out.println("================================================================"); + } +} diff --git a/rocketmq-store/src/test/java/com/alibaba/rocketmq/store/MapedFileQueueTest.java b/rocketmq-store/src/test/java/com/alibaba/rocketmq/store/MapedFileQueueTest.java new file mode 100644 index 000000000..8f89e41c4 --- /dev/null +++ b/rocketmq-store/src/test/java/com/alibaba/rocketmq/store/MapedFileQueueTest.java @@ -0,0 +1,206 @@ +/** + * $Id: MapedFileQueueTest.java 1831 2013-05-16 01:39:51Z shijia.wxr $ + */ +package com.alibaba.rocketmq.store; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + + +public class MapedFileQueueTest { + + // private static final String StoreMessage = + // "Once, there was a chance for me! but I did not treasure it. if"; + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + } + + + @AfterClass + public static void tearDownAfterClass() throws Exception { + } + + + @Before + public void setUp() throws Exception { + } + + + @After + public void tearDown() throws Exception { + } + + + @Test + public void test_getLastMapedFile() { + final String fixedMsg = "0123456789abcdef"; + System.out.println("================================================================"); + AllocateMapedFileService allocateMapedFileService = new AllocateMapedFileService(); + allocateMapedFileService.start(); + MapedFileQueue mapedFileQueue = + new MapedFileQueue("./unit_test_store/a/", 1024, allocateMapedFileService); + + for (int i = 0; i < 1024; i++) { + MapedFile mapedFile = mapedFileQueue.getLastMapedFile(); + assertTrue(mapedFile != null); + boolean result = mapedFile.appendMessage(fixedMsg.getBytes()); + if (!result) { + System.out.println("appendMessage " + i); + } + assertTrue(result); + } + + mapedFileQueue.shutdown(1000); + mapedFileQueue.destroy(); + allocateMapedFileService.shutdown(); + System.out.println("MapedFileQueue.getLastMapedFile() OK"); + } + + + @Test + public void test_findMapedFileByOffset() { + final String fixedMsg = "abcd"; + System.out.println("================================================================"); + AllocateMapedFileService allocateMapedFileService = new AllocateMapedFileService(); + allocateMapedFileService.start(); + MapedFileQueue mapedFileQueue = + new MapedFileQueue("./unit_test_store/b/", 1024, allocateMapedFileService); + + for (int i = 0; i < 1024; i++) { + MapedFile mapedFile = mapedFileQueue.getLastMapedFile(); + assertTrue(mapedFile != null); + boolean result = mapedFile.appendMessage(fixedMsg.getBytes()); + // System.out.println("appendMessage " + bytes); + assertTrue(result); + } + + MapedFile mapedFile = mapedFileQueue.findMapedFileByOffset(0); + assertTrue(mapedFile != null); + assertEquals(mapedFile.getFileFromOffset(), 0); + System.out.println(mapedFile.getFileFromOffset()); + + mapedFile = mapedFileQueue.findMapedFileByOffset(100); + assertTrue(mapedFile != null); + assertEquals(mapedFile.getFileFromOffset(), 0); + System.out.println(mapedFile.getFileFromOffset()); + + mapedFile = mapedFileQueue.findMapedFileByOffset(1024); + assertTrue(mapedFile != null); + assertEquals(mapedFile.getFileFromOffset(), 1024); + System.out.println(mapedFile.getFileFromOffset()); + + mapedFile = mapedFileQueue.findMapedFileByOffset(1024 + 100); + assertTrue(mapedFile != null); + assertEquals(mapedFile.getFileFromOffset(), 1024); + System.out.println(mapedFile.getFileFromOffset()); + + mapedFile = mapedFileQueue.findMapedFileByOffset(1024 * 2); + assertTrue(mapedFile != null); + assertEquals(mapedFile.getFileFromOffset(), 1024 * 2); + System.out.println(mapedFile.getFileFromOffset()); + + mapedFile = mapedFileQueue.findMapedFileByOffset(1024 * 2 + 100); + assertTrue(mapedFile != null); + assertEquals(mapedFile.getFileFromOffset(), 1024 * 2); + System.out.println(mapedFile.getFileFromOffset()); + + mapedFile = mapedFileQueue.findMapedFileByOffset(1024 * 4); + assertTrue(mapedFile == null); + + mapedFile = mapedFileQueue.findMapedFileByOffset(1024 * 4 + 100); + assertTrue(mapedFile == null); + + mapedFileQueue.shutdown(1000); + mapedFileQueue.destroy(); + allocateMapedFileService.shutdown(); + System.out.println("MapedFileQueue.findMapedFileByOffset() OK"); + } + + + @Test + public void test_commit() { + final String fixedMsg = "0123456789abcdef"; + System.out.println("================================================================"); + AllocateMapedFileService allocateMapedFileService = new AllocateMapedFileService(); + allocateMapedFileService.start(); + MapedFileQueue mapedFileQueue = + new MapedFileQueue("./unit_test_store/c/", 1024, allocateMapedFileService); + + for (int i = 0; i < 1024; i++) { + MapedFile mapedFile = mapedFileQueue.getLastMapedFile(); + assertTrue(mapedFile != null); + boolean result = mapedFile.appendMessage(fixedMsg.getBytes()); + assertTrue(result); + } + + // 不断尝试提交 + boolean result = mapedFileQueue.commit(0); + assertFalse(result); + assertEquals(1024 * 1, mapedFileQueue.getCommittedWhere()); + System.out.println("1 " + result + " " + mapedFileQueue.getCommittedWhere()); + + result = mapedFileQueue.commit(0); + assertFalse(result); + assertEquals(1024 * 2, mapedFileQueue.getCommittedWhere()); + System.out.println("2 " + result + " " + mapedFileQueue.getCommittedWhere()); + + result = mapedFileQueue.commit(0); + assertFalse(result); + assertEquals(1024 * 3, mapedFileQueue.getCommittedWhere()); + System.out.println("3 " + result + " " + mapedFileQueue.getCommittedWhere()); + + result = mapedFileQueue.commit(0); + assertFalse(result); + assertEquals(1024 * 4, mapedFileQueue.getCommittedWhere()); + System.out.println("4 " + result + " " + mapedFileQueue.getCommittedWhere()); + + result = mapedFileQueue.commit(0); + assertFalse(result); + assertEquals(1024 * 5, mapedFileQueue.getCommittedWhere()); + System.out.println("5 " + result + " " + mapedFileQueue.getCommittedWhere()); + + result = mapedFileQueue.commit(0); + assertFalse(result); + assertEquals(1024 * 6, mapedFileQueue.getCommittedWhere()); + System.out.println("6 " + result + " " + mapedFileQueue.getCommittedWhere()); + + mapedFileQueue.shutdown(1000); + mapedFileQueue.destroy(); + allocateMapedFileService.shutdown(); + System.out.println("MapedFileQueue.commit() OK"); + } + + + @Test + public void test_getMapedMemorySize() { + final String fixedMsg = "abcd"; + System.out.println("================================================================"); + AllocateMapedFileService allocateMapedFileService = new AllocateMapedFileService(); + allocateMapedFileService.start(); + MapedFileQueue mapedFileQueue = + new MapedFileQueue("./unit_test_store/d/", 1024, allocateMapedFileService); + + for (int i = 0; i < 1024; i++) { + MapedFile mapedFile = mapedFileQueue.getLastMapedFile(); + assertTrue(mapedFile != null); + boolean result = mapedFile.appendMessage(fixedMsg.getBytes()); + assertTrue(result); + } + + assertEquals(fixedMsg.length() * 1024, mapedFileQueue.getMapedMemorySize()); + + mapedFileQueue.shutdown(1000); + mapedFileQueue.destroy(); + allocateMapedFileService.shutdown(); + System.out.println("MapedFileQueue.getMapedMemorySize() OK"); + } + +} diff --git a/rocketmq-store/src/test/java/com/alibaba/rocketmq/store/MapedFileTest.java b/rocketmq-store/src/test/java/com/alibaba/rocketmq/store/MapedFileTest.java new file mode 100644 index 000000000..983c1a388 --- /dev/null +++ b/rocketmq-store/src/test/java/com/alibaba/rocketmq/store/MapedFileTest.java @@ -0,0 +1,94 @@ +/** + * $Id: MapedFileTest.java 1831 2013-05-16 01:39:51Z shijia.wxr $ + */ +package com.alibaba.rocketmq.store; + +import static org.junit.Assert.assertTrue; + +import java.io.IOException; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Ignore; +import org.junit.Test; + + +public class MapedFileTest { + + private static final String StoreMessage = "Once, there was a chance for me!"; + + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + + } + + + @AfterClass + public static void tearDownAfterClass() throws Exception { + } + + + @Test + public void test_write_read() { + try { + MapedFile mapedFile = new MapedFile("./unit_test_store/MapedFileTest/000", 1024 * 64); + boolean result = mapedFile.appendMessage(StoreMessage.getBytes()); + assertTrue(result); + System.out.println("write OK"); + + SelectMapedBufferResult selectMapedBufferResult = mapedFile.selectMapedBuffer(0); + byte[] data = new byte[StoreMessage.length()]; + selectMapedBufferResult.getByteBuffer().get(data); + String readString = new String(data); + + System.out.println("Read: " + readString); + assertTrue(readString.equals(StoreMessage)); + + // 禁止Buffer读写 + mapedFile.shutdown(1000); + + // mapedFile对象不可用 + assertTrue(!mapedFile.isAvailable()); + + // 释放读到的Buffer + selectMapedBufferResult.release(); + + // 内存真正释放掉 + assertTrue(mapedFile.isCleanupOver()); + + // 文件删除成功 + assertTrue(mapedFile.destroy(1000)); + } + catch (IOException e) { + e.printStackTrace(); + } + } + + + /** + * 当前测试用例由于对mmap操作错误，会导致JVM CRASHED + */ + @Ignore + public void test_jvm_crashed() { + try { + MapedFile mapedFile = new MapedFile("./unit_test_store/MapedFileTest/10086", 1024 * 64); + boolean result = mapedFile.appendMessage(StoreMessage.getBytes()); + assertTrue(result); + System.out.println("write OK"); + + SelectMapedBufferResult selectMapedBufferResult = mapedFile.selectMapedBuffer(0); + selectMapedBufferResult.release(); + mapedFile.shutdown(1000); + + byte[] data = new byte[StoreMessage.length()]; + selectMapedBufferResult.getByteBuffer().get(data); + String readString = new String(data); + System.out.println(readString); + } + catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } +} diff --git a/rocketmq-store/src/test/java/com/alibaba/rocketmq/store/RecoverTest.java b/rocketmq-store/src/test/java/com/alibaba/rocketmq/store/RecoverTest.java new file mode 100644 index 000000000..62897d790 --- /dev/null +++ b/rocketmq-store/src/test/java/com/alibaba/rocketmq/store/RecoverTest.java @@ -0,0 +1,253 @@ +/** + * $Id: RecoverTest.java 1831 2013-05-16 01:39:51Z shijia.wxr $ + */ +package com.alibaba.rocketmq.store; + +import com.alibaba.rocketmq.common.message.MessageDecoder; +import com.alibaba.rocketmq.common.message.MessageExt; +import com.alibaba.rocketmq.store.config.MessageStoreConfig; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.nio.ByteBuffer; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.Assert.assertTrue; + + +public class RecoverTest { + // 队列个数 + private static int QUEUE_TOTAL = 10; + // 发往哪个队列 + private static AtomicInteger QueueId = new AtomicInteger(0); + // 发送主机地址 + private static SocketAddress BornHost; + // 存储主机地址 + private static SocketAddress StoreHost; + // 消息体 + private static byte[] MessageBody; + + private static final String StoreMessage = "Once, there was a chance for me!aaaaaaaaaaaaaaaaaaaaaaaa"; + + + public MessageExtBrokerInner buildMessage() { + MessageExtBrokerInner msg = new MessageExtBrokerInner(); + msg.setTopic("TOPIC_A"); + msg.setTags("TAG1"); + msg.setKeys("Hello"); + msg.setBody(MessageBody); + msg.setKeys(String.valueOf(System.currentTimeMillis())); + msg.setQueueId(Math.abs(QueueId.getAndIncrement()) % QUEUE_TOTAL); + msg.setSysFlag(4); + msg.setBornTimestamp(System.currentTimeMillis()); + msg.setStoreHost(StoreHost); + msg.setBornHost(BornHost); + + return msg; + } + + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + StoreHost = new InetSocketAddress(InetAddress.getLocalHost(), 8123); + BornHost = new InetSocketAddress(InetAddress.getByName("127.0.0.1"), 0); + } + + + @AfterClass + public static void tearDownAfterClass() throws Exception { + } + + private MessageStore storeWrite1; + private MessageStore storeWrite2; + private MessageStore storeRead; + + + private void destroy() { + if (storeWrite1 != null) { + // 关闭存储服务 + storeWrite1.shutdown(); + // 删除文件 + storeWrite1.destroy(); + } + + if (storeWrite2 != null) { + // 关闭存储服务 + storeWrite2.shutdown(); + // 删除文件 + storeWrite2.destroy(); + } + + if (storeRead != null) { + // 关闭存储服务 + storeRead.shutdown(); + // 删除文件 + storeRead.destroy(); + } + } + + + public void writeMessage(boolean normal, boolean first) throws Exception { + System.out.println("================================================================"); + long totalMsgs = 1000; + QUEUE_TOTAL = 3; + + // 构造消息体 + MessageBody = StoreMessage.getBytes(); + + MessageStoreConfig messageStoreConfig = new MessageStoreConfig(); + // 每个物理映射文件 + messageStoreConfig.setMapedFileSizeCommitLog(1024 * 32); + // 每个逻辑映射文件 + messageStoreConfig.setMapedFileSizeConsumeQueue(100 * 20); + messageStoreConfig.setMessageIndexEnable(false); + + MessageStore messageStore = new DefaultMessageStore(messageStoreConfig, null); + if (first) { + this.storeWrite1 = messageStore; + } + else { + this.storeWrite2 = messageStore; + } + + // 第一步，load已有数据 + boolean loadResult = messageStore.load(); + assertTrue(loadResult); + + // 第二步，启动服务 + messageStore.start(); + + // 第三步，发消息 + for (long i = 0; i < totalMsgs; i++) { + + PutMessageResult result = messageStore.putMessage(buildMessage()); + + System.out.println(i + "\t" + result.getAppendMessageResult().getMsgId()); + } + + if (normal) { + // 关闭存储服务 + messageStore.shutdown(); + } + + System.out.println("========================writeMessage OK========================================"); + } + + + private void veryReadMessage(int queueId, long queueOffset, List byteBuffers) { + for (ByteBuffer byteBuffer : byteBuffers) { + MessageExt msg = MessageDecoder.decode(byteBuffer); + System.out.println("request queueId " + queueId + ", request queueOffset " + queueOffset + + " msg queue offset " + msg.getQueueOffset()); + + assertTrue(msg.getQueueOffset() == queueOffset); + + queueOffset++; + } + } + + + public void readMessage(final long msgCnt) throws Exception { + System.out.println("================================================================"); + QUEUE_TOTAL = 3; + + // 构造消息体 + MessageBody = StoreMessage.getBytes(); + + MessageStoreConfig messageStoreConfig = new MessageStoreConfig(); + // 每个物理映射文件 + messageStoreConfig.setMapedFileSizeCommitLog(1024 * 32); + // 每个逻辑映射文件 + messageStoreConfig.setMapedFileSizeConsumeQueue(100 * 20); + messageStoreConfig.setMessageIndexEnable(false); + + storeRead = new DefaultMessageStore(messageStoreConfig, null); + // 第一步，load已有数据 + boolean loadResult = storeRead.load(); + assertTrue(loadResult); + + // 第二步，启动服务 + storeRead.start(); + + // 第三步，收消息 + long readCnt = 0; + for (int queueId = 0; queueId < QUEUE_TOTAL; queueId++) { + for (long offset = 0;;) { + GetMessageResult result = + storeRead.getMessage("GROUP_A", "TOPIC_A", queueId, offset, 1024 * 1024, null); + if (result.getStatus() == GetMessageStatus.FOUND) { + System.out.println(queueId + "\t" + result.getMessageCount()); + this.veryReadMessage(queueId, offset, result.getMessageBufferList()); + offset += result.getMessageCount(); + readCnt += result.getMessageCount(); + result.release(); + } + else { + break; + } + } + } + + System.out.println("readCnt = " + readCnt); + assertTrue(readCnt == msgCnt); + + System.out.println("========================readMessage OK========================================"); + } + + + /** + * 正常关闭后，重启恢复消息，验证是否有消息丢失 + */ + @Test + public void test_recover_normally() throws Exception { + this.writeMessage(true, true); + Thread.sleep(1000 * 3); + this.readMessage(1000); + this.destroy(); + } + + + /** + * 正常关闭后，重启恢复消息，并再次写入消息，验证是否有消息丢失 + */ + @Test + public void test_recover_normally_write() throws Exception { + this.writeMessage(true, true); + Thread.sleep(1000 * 3); + this.writeMessage(true, false); + Thread.sleep(1000 * 3); + this.readMessage(2000); + this.destroy(); + } + + + /** + * 异常关闭后，重启恢复消息，验证是否有消息丢失 + */ + @Test + public void test_recover_abnormally() throws Exception { + this.writeMessage(false, true); + Thread.sleep(1000 * 3); + this.readMessage(1000); + this.destroy(); + } + + + /** + * 异常关闭后，重启恢复消息，并再次写入消息，验证是否有消息丢失 + */ + @Test + public void test_recover_abnormally_write() throws Exception { + this.writeMessage(false, true); + Thread.sleep(1000 * 3); + this.writeMessage(false, false); + Thread.sleep(1000 * 3); + this.readMessage(2000); + this.destroy(); + } +} diff --git a/rocketmq-store/src/test/java/com/alibaba/rocketmq/store/StoreCheckpointTest.java b/rocketmq-store/src/test/java/com/alibaba/rocketmq/store/StoreCheckpointTest.java new file mode 100644 index 000000000..034ac9013 --- /dev/null +++ b/rocketmq-store/src/test/java/com/alibaba/rocketmq/store/StoreCheckpointTest.java @@ -0,0 +1,51 @@ +/** + * $Id: StoreCheckpointTest.java 1831 2013-05-16 01:39:51Z shijia.wxr $ + */ +package com.alibaba.rocketmq.store; + +import static org.junit.Assert.assertTrue; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + + +public class StoreCheckpointTest { + @BeforeClass + public static void setUpBeforeClass() throws Exception { + + } + + + @AfterClass + public static void tearDownAfterClass() throws Exception { + } + + + @Test + public void test_write_read() { + try { + StoreCheckpoint storeCheckpoint = new StoreCheckpoint("./a/b/0000"); + long physicMsgTimestamp = 0xAABB; + long logicsMsgTimestamp = 0xCCDD; + storeCheckpoint.setPhysicMsgTimestamp(physicMsgTimestamp); + storeCheckpoint.setLogicsMsgTimestamp(logicsMsgTimestamp); + storeCheckpoint.flush(); + + // 因为时间精度问题，所以最小时间向前回退3s + long diff = physicMsgTimestamp - storeCheckpoint.getMinTimestamp(); + assertTrue(diff == 3000); + + storeCheckpoint.shutdown(); + + storeCheckpoint = new StoreCheckpoint("a/b/0000"); + assertTrue(physicMsgTimestamp == storeCheckpoint.getPhysicMsgTimestamp()); + assertTrue(logicsMsgTimestamp == storeCheckpoint.getLogicsMsgTimestamp()); + } + catch (Throwable e) { + e.printStackTrace(); + assertTrue(false); + } + + } +} diff --git a/rocketmq-store/src/test/java/com/alibaba/rocketmq/store/index/IndexFileTest.java b/rocketmq-store/src/test/java/com/alibaba/rocketmq/store/index/IndexFileTest.java new file mode 100644 index 000000000..9c4eaf82f --- /dev/null +++ b/rocketmq-store/src/test/java/com/alibaba/rocketmq/store/index/IndexFileTest.java @@ -0,0 +1,77 @@ +/** + * $Id: IndexFileTest.java 1831 2013-05-16 01:39:51Z shijia.wxr $ + */ +package com.alibaba.rocketmq.store.index; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.util.ArrayList; +import java.util.List; + +import org.junit.Test; + + +public class IndexFileTest { + private final int hashSlotNum = 100; + private final int indexNum = 400; + + + @Test + public void test_put_index() { + try { + IndexFile indexFile = new IndexFile("100", hashSlotNum, indexNum, 0, 0); + + // 写入索引 + for (long i = 0; i < (indexNum - 1); i++) { + boolean putResult = indexFile.putKey(Long.toString(i), i, System.currentTimeMillis()); + assertTrue(putResult); + } + + // 索引文件已经满了， 再写入会失败 + boolean putResult = indexFile.putKey(Long.toString(400), 400, System.currentTimeMillis()); + assertFalse(putResult); + + // 删除文件 + indexFile.destroy(0); + } + catch (Exception e) { + e.printStackTrace(); + assertTrue(false); + } + } + + + @Test + public void test_put_get_index() { + try { + IndexFile indexFile = new IndexFile("200", hashSlotNum, indexNum, 0, 0); + + // 写入索引 + for (long i = 0; i < (indexNum - 1); i++) { + boolean putResult = indexFile.putKey(Long.toString(i), i, System.currentTimeMillis()); + assertTrue(putResult); + } + + // 索引文件已经满了， 再写入会失败 + boolean putResult = indexFile.putKey(Long.toString(400), 400, System.currentTimeMillis()); + assertFalse(putResult); + + // 读索引 + final List phyOffsets = new ArrayList(); + indexFile.selectPhyOffset(phyOffsets, "60", 10, 0, Long.MAX_VALUE, true); + for (Long offset : phyOffsets) { + System.out.println(offset); + } + + assertFalse(phyOffsets.isEmpty()); + + // 删除文件 + indexFile.destroy(0); + } + catch (Exception e) { + e.printStackTrace(); + assertTrue(false); + } + } +} diff --git a/rocketmq-store/src/test/java/com/alibaba/rocketmq/store/schedule/ScheduleMessageTest.java b/rocketmq-store/src/test/java/com/alibaba/rocketmq/store/schedule/ScheduleMessageTest.java new file mode 100644 index 000000000..47a52dc83 --- /dev/null +++ b/rocketmq-store/src/test/java/com/alibaba/rocketmq/store/schedule/ScheduleMessageTest.java @@ -0,0 +1,124 @@ +/** + * $Id: ScheduleMessageTest.java 1831 2013-05-16 01:39:51Z shijia.wxr $ + */ +package com.alibaba.rocketmq.store.schedule; + +import com.alibaba.rocketmq.store.*; +import com.alibaba.rocketmq.store.config.MessageStoreConfig; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.Assert.assertTrue; + + +public class ScheduleMessageTest { + // 队列个数 + private static int QUEUE_TOTAL = 100; + // 发往哪个队列 + private static AtomicInteger QueueId = new AtomicInteger(0); + // 发送主机地址 + private static SocketAddress BornHost; + // 存储主机地址 + private static SocketAddress StoreHost; + // 消息体 + private static byte[] MessageBody; + + private static final String StoreMessage = "Once, there was a chance for me!"; + + + public MessageExtBrokerInner buildMessage() { + MessageExtBrokerInner msg = new MessageExtBrokerInner(); + msg.setTopic("AAA"); + msg.setTags("TAG1"); + msg.setKeys("Hello"); + msg.setBody(MessageBody); + msg.setKeys(String.valueOf(System.currentTimeMillis())); + msg.setQueueId(Math.abs(QueueId.getAndIncrement()) % QUEUE_TOTAL); + msg.setSysFlag(4); + msg.setBornTimestamp(System.currentTimeMillis()); + msg.setStoreHost(StoreHost); + msg.setBornHost(BornHost); + + return msg; + } + + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + StoreHost = new InetSocketAddress(InetAddress.getLocalHost(), 8123); + BornHost = new InetSocketAddress(InetAddress.getByName("127.0.0.1"), 0); + } + + + @AfterClass + public static void tearDownAfterClass() throws Exception { + } + + + @Test + public void test_delay_message() throws Exception { + System.out.println("================================================================"); + long totalMsgs = 10000; + QUEUE_TOTAL = 32; + + // 构造消息体 + MessageBody = StoreMessage.getBytes(); + + MessageStoreConfig messageStoreConfig = new MessageStoreConfig(); + // 每个物理映射文件 4K + messageStoreConfig.setMapedFileSizeCommitLog(1024 * 32); + messageStoreConfig.setMapedFileSizeConsumeQueue(1024 * 16); + messageStoreConfig.setMaxHashSlotNum(100); + messageStoreConfig.setMaxIndexNum(1000 * 10); + + MessageStore master = new DefaultMessageStore(messageStoreConfig, null); + // 第一步，load已有数据 + boolean load = master.load(); + assertTrue(load); + + // 第二步，启动服务 + master.start(); + for (int i = 0; i < totalMsgs; i++) { + MessageExtBrokerInner msg = buildMessage(); + msg.setDelayTimeLevel(i % 4); + + PutMessageResult result = master.putMessage(msg); + System.out.println(i + "\t" + result.getAppendMessageResult().getMsgId()); + } + + System.out.println("write message over, wait time up"); + Thread.sleep(1000 * 20); + + // 开始读文件 + for (long i = 0; i < totalMsgs; i++) { + try { + GetMessageResult result = master.getMessage("GROUP_A", "TOPIC_A", 0, i, 1024 * 1024, null); + if (result == null) { + System.out.println("result == null " + i); + } + assertTrue(result != null); + result.release(); + System.out.println("read " + i + " OK"); + } + catch (Exception e) { + e.printStackTrace(); + } + + } + + Thread.sleep(1000 * 15); + + // 关闭存储服务 + master.shutdown(); + + // 删除文件 + master.destroy(); + System.out.println("================================================================"); + } +} diff --git a/rocketmq-tools/pom.xml b/rocketmq-tools/pom.xml new file mode 100644 index 000000000..3cbd9fa3f --- /dev/null +++ b/rocketmq-tools/pom.xml @@ -0,0 +1,45 @@ + + + com.alibaba.rocketmq + rocketmq-all + 3.2.2 + + + 4.0.0 + jar + rocketmq-tools + rocketmq-tools ${project.version} + + + + junit + junit + test + + + ${project.groupId} + rocketmq-client + + + ${project.groupId} + rocketmq-store + + + ${project.groupId} + rocketmq-srvutil + + + com.alibaba + fastjson + + + ch.qos.logback + logback-classic + + + ch.qos.logback + logback-core + + + diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/admin/DefaultMQAdminExt.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/admin/DefaultMQAdminExt.java new file mode 100644 index 000000000..7e3f238f4 --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/admin/DefaultMQAdminExt.java @@ -0,0 +1,436 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.tools.admin; + +import java.io.UnsupportedEncodingException; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; + +import com.alibaba.rocketmq.client.ClientConfig; +import com.alibaba.rocketmq.client.QueryResult; +import com.alibaba.rocketmq.client.exception.MQBrokerException; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.TopicConfig; +import com.alibaba.rocketmq.common.admin.ConsumeStats; +import com.alibaba.rocketmq.common.admin.RollbackStats; +import com.alibaba.rocketmq.common.admin.TopicStatsTable; +import com.alibaba.rocketmq.common.message.MessageExt; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.common.protocol.body.ClusterInfo; +import com.alibaba.rocketmq.common.protocol.body.ConsumeMessageDirectlyResult; +import com.alibaba.rocketmq.common.protocol.body.ConsumerConnection; +import com.alibaba.rocketmq.common.protocol.body.ConsumerRunningInfo; +import com.alibaba.rocketmq.common.protocol.body.GroupList; +import com.alibaba.rocketmq.common.protocol.body.KVTable; +import com.alibaba.rocketmq.common.protocol.body.ProducerConnection; +import com.alibaba.rocketmq.common.protocol.body.QueueTimeSpan; +import com.alibaba.rocketmq.common.protocol.body.TopicList; +import com.alibaba.rocketmq.common.protocol.route.TopicRouteData; +import com.alibaba.rocketmq.common.subscription.SubscriptionGroupConfig; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; +import com.alibaba.rocketmq.remoting.exception.RemotingConnectException; +import com.alibaba.rocketmq.remoting.exception.RemotingException; +import com.alibaba.rocketmq.remoting.exception.RemotingSendRequestException; +import com.alibaba.rocketmq.remoting.exception.RemotingTimeoutException; +import com.alibaba.rocketmq.tools.admin.api.MessageTrack; + + +/** + * 所有运维接口都在这里实现 + * + * @author shijia.wxr + * @since 2013-7-14 + */ +public class DefaultMQAdminExt extends ClientConfig implements MQAdminExt { + private final DefaultMQAdminExtImpl defaultMQAdminExtImpl; + private String adminExtGroup = "admin_ext_group"; + private String createTopicKey = MixAll.DEFAULT_TOPIC; + + + public DefaultMQAdminExt() { + this.defaultMQAdminExtImpl = new DefaultMQAdminExtImpl(this, null); + } + + + public DefaultMQAdminExt(RPCHook rpcHook) { + this.defaultMQAdminExtImpl = new DefaultMQAdminExtImpl(this, rpcHook); + } + + + public DefaultMQAdminExt(final String adminExtGroup) { + this.adminExtGroup = adminExtGroup; + this.defaultMQAdminExtImpl = new DefaultMQAdminExtImpl(this); + } + + + @Override + public void createTopic(String key, String newTopic, int queueNum) throws MQClientException { + createTopic(key, newTopic, queueNum, 0); + } + + + @Override + public void createTopic(String key, String newTopic, int queueNum, int topicSysFlag) + throws MQClientException { + defaultMQAdminExtImpl.createTopic(key, newTopic, queueNum, topicSysFlag); + } + + + @Override + public long searchOffset(MessageQueue mq, long timestamp) throws MQClientException { + return defaultMQAdminExtImpl.searchOffset(mq, timestamp); + } + + + @Override + public long maxOffset(MessageQueue mq) throws MQClientException { + return defaultMQAdminExtImpl.maxOffset(mq); + } + + + @Override + public long minOffset(MessageQueue mq) throws MQClientException { + return defaultMQAdminExtImpl.minOffset(mq); + } + + + @Override + public long earliestMsgStoreTime(MessageQueue mq) throws MQClientException { + return defaultMQAdminExtImpl.earliestMsgStoreTime(mq); + } + + + @Override + public MessageExt viewMessage(String msgId) throws RemotingException, MQBrokerException, + InterruptedException, MQClientException { + return defaultMQAdminExtImpl.viewMessage(msgId); + } + + + @Override + public QueryResult queryMessage(String topic, String key, int maxNum, long begin, long end) + throws MQClientException, InterruptedException { + return defaultMQAdminExtImpl.queryMessage(topic, key, maxNum, begin, end); + } + + + @Override + public void start() throws MQClientException { + defaultMQAdminExtImpl.start(); + } + + + @Override + public void shutdown() { + defaultMQAdminExtImpl.shutdown(); + } + + + @Override + public void createAndUpdateTopicConfig(String addr, TopicConfig config) throws RemotingException, + MQBrokerException, InterruptedException, MQClientException { + defaultMQAdminExtImpl.createAndUpdateTopicConfig(addr, config); + } + + + @Override + public void createAndUpdateSubscriptionGroupConfig(String addr, SubscriptionGroupConfig config) + throws RemotingException, MQBrokerException, InterruptedException, MQClientException { + defaultMQAdminExtImpl.createAndUpdateSubscriptionGroupConfig(addr, config); + } + + + @Override + public SubscriptionGroupConfig examineSubscriptionGroupConfig(String addr, String group) { + return defaultMQAdminExtImpl.examineSubscriptionGroupConfig(addr, group); + } + + + @Override + public TopicConfig examineTopicConfig(String addr, String topic) { + return defaultMQAdminExtImpl.examineTopicConfig(addr, topic); + } + + + @Override + public TopicStatsTable examineTopicStats(String topic) throws RemotingException, MQClientException, + InterruptedException, MQBrokerException { + return defaultMQAdminExtImpl.examineTopicStats(topic); + } + + + @Override + public ConsumeStats examineConsumeStats(String consumerGroup) throws RemotingException, + MQClientException, InterruptedException, MQBrokerException { + return defaultMQAdminExtImpl.examineConsumeStats(consumerGroup); + } + + + @Override + public ClusterInfo examineBrokerClusterInfo() throws InterruptedException, RemotingConnectException, + RemotingTimeoutException, RemotingSendRequestException, MQBrokerException { + return defaultMQAdminExtImpl.examineBrokerClusterInfo(); + } + + + @Override + public TopicRouteData examineTopicRouteInfo(String topic) throws RemotingException, MQClientException, + InterruptedException { + return defaultMQAdminExtImpl.examineTopicRouteInfo(topic); + } + + + @Override + public void putKVConfig(String namespace, String key, String value) { + defaultMQAdminExtImpl.putKVConfig(namespace, key, value); + } + + + @Override + public String getKVConfig(String namespace, String key) throws RemotingException, MQClientException, + InterruptedException { + return defaultMQAdminExtImpl.getKVConfig(namespace, key); + } + + + @Override + public ConsumerConnection examineConsumerConnectionInfo(String consumerGroup) + throws InterruptedException, MQBrokerException, RemotingException, MQClientException { + return defaultMQAdminExtImpl.examineConsumerConnectionInfo(consumerGroup); + } + + + @Override + public ProducerConnection examineProducerConnectionInfo(String producerGroup, final String topic) + throws RemotingException, MQClientException, InterruptedException, MQBrokerException { + return defaultMQAdminExtImpl.examineProducerConnectionInfo(producerGroup, topic); + } + + + @Override + public int wipeWritePermOfBroker(final String namesrvAddr, String brokerName) + throws RemotingCommandException, RemotingConnectException, RemotingSendRequestException, + RemotingTimeoutException, InterruptedException, MQClientException { + return defaultMQAdminExtImpl.wipeWritePermOfBroker(namesrvAddr, brokerName); + } + + + public String getAdminExtGroup() { + return adminExtGroup; + } + + + public void setAdminExtGroup(String adminExtGroup) { + this.adminExtGroup = adminExtGroup; + } + + + public String getCreateTopicKey() { + return createTopicKey; + } + + + public void setCreateTopicKey(String createTopicKey) { + this.createTopicKey = createTopicKey; + } + + + @Override + public List getNameServerAddressList() { + return this.defaultMQAdminExtImpl.getNameServerAddressList(); + } + + + @Override + public TopicList fetchAllTopicList() throws RemotingException, MQClientException, InterruptedException { + return this.defaultMQAdminExtImpl.fetchAllTopicList(); + } + + + @Override + public KVTable fetchBrokerRuntimeStats(final String brokerAddr) throws RemotingConnectException, + RemotingSendRequestException, RemotingTimeoutException, InterruptedException, MQBrokerException { + return this.defaultMQAdminExtImpl.fetchBrokerRuntimeStats(brokerAddr); + } + + + @Override + public void deleteTopicInBroker(Set addrs, String topic) throws RemotingException, + MQBrokerException, InterruptedException, MQClientException { + defaultMQAdminExtImpl.deleteTopicInBroker(addrs, topic); + } + + + @Override + public void deleteTopicInNameServer(Set addrs, String topic) throws RemotingException, + MQBrokerException, InterruptedException, MQClientException { + defaultMQAdminExtImpl.deleteTopicInNameServer(addrs, topic); + } + + + @Override + public void deleteSubscriptionGroup(String addr, String groupName) throws RemotingException, + MQBrokerException, InterruptedException, MQClientException { + defaultMQAdminExtImpl.deleteSubscriptionGroup(addr, groupName); + } + + + @Override + public void createAndUpdateKvConfig(String namespace, String key, String value) throws RemotingException, + MQBrokerException, InterruptedException, MQClientException { + defaultMQAdminExtImpl.createAndUpdateKvConfig(namespace, key, value); + } + + + @Override + public void deleteKvConfig(String namespace, String key) throws RemotingException, MQBrokerException, + InterruptedException, MQClientException { + defaultMQAdminExtImpl.deleteKvConfig(namespace, key); + } + + + @Override + public String getProjectGroupByIp(String ip) throws RemotingException, MQBrokerException, + InterruptedException, MQClientException { + return defaultMQAdminExtImpl.getProjectGroupByIp(ip); + } + + + @Override + public String getIpsByProjectGroup(String projectGroup) throws RemotingException, MQBrokerException, + InterruptedException, MQClientException { + return defaultMQAdminExtImpl.getIpsByProjectGroup(projectGroup); + } + + + @Override + public void deleteIpsByProjectGroup(String projectGroup) throws RemotingException, MQBrokerException, + InterruptedException, MQClientException { + defaultMQAdminExtImpl.deleteIpsByProjectGroup(projectGroup); + } + + + public List resetOffsetByTimestampOld(String consumerGroup, String topic, long timestamp, + boolean force) throws RemotingException, MQBrokerException, InterruptedException, + MQClientException { + return defaultMQAdminExtImpl.resetOffsetByTimestampOld(consumerGroup, topic, timestamp, force); + } + + + @Override + public KVTable getKVListByNamespace(String namespace) throws RemotingException, MQClientException, + InterruptedException { + return defaultMQAdminExtImpl.getKVListByNamespace(namespace); + } + + + @Override + public void updateBrokerConfig(String brokerAddr, Properties properties) throws RemotingConnectException, + RemotingSendRequestException, RemotingTimeoutException, UnsupportedEncodingException, + InterruptedException, MQBrokerException { + defaultMQAdminExtImpl.updateBrokerConfig(brokerAddr, properties); + } + + + @Override + public Map resetOffsetByTimestamp(String topic, String group, long timestamp, + boolean isForce) throws RemotingException, MQBrokerException, InterruptedException, + MQClientException { + return defaultMQAdminExtImpl.resetOffsetByTimestamp(topic, group, timestamp, isForce); + } + + + @Override + public Map> getConsumeStatus(String topic, String group, String clientAddr) + throws RemotingException, MQBrokerException, InterruptedException, MQClientException { + return defaultMQAdminExtImpl.getConsumeStatus(topic, group, clientAddr); + } + + + @Override + public void createOrUpdateOrderConf(String key, String value, boolean isCluster) + throws RemotingException, MQBrokerException, InterruptedException, MQClientException { + defaultMQAdminExtImpl.createOrUpdateOrderConf(key, value, isCluster); + } + + + @Override + public GroupList queryTopicConsumeByWho(String topic) throws InterruptedException, MQBrokerException, + RemotingException, MQClientException { + return this.defaultMQAdminExtImpl.queryTopicConsumeByWho(topic); + } + + + @Override + public Set queryConsumeTimeSpan(final String topic, final String group) + throws InterruptedException, MQBrokerException, RemotingException, MQClientException { + return this.defaultMQAdminExtImpl.queryConsumeTimeSpan(topic, group); + } + + + @Override + public void resetOffsetNew(String consumerGroup, String topic, long timestamp) throws RemotingException, + MQBrokerException, InterruptedException, MQClientException { + this.defaultMQAdminExtImpl.resetOffsetNew(consumerGroup, topic, timestamp); + } + + + @Override + public boolean cleanExpiredConsumerQueue(String cluster) throws RemotingConnectException, + RemotingSendRequestException, RemotingTimeoutException, MQClientException, InterruptedException { + return defaultMQAdminExtImpl.cleanExpiredConsumerQueue(cluster); + } + + + @Override + public boolean cleanExpiredConsumerQueueByAddr(String addr) throws RemotingConnectException, + RemotingSendRequestException, RemotingTimeoutException, MQClientException, InterruptedException { + return defaultMQAdminExtImpl.cleanExpiredConsumerQueueByAddr(addr); + } + + + @Override + public ConsumerRunningInfo getConsumerRunningInfo(String consumerGroup, String clientId, boolean jstack) + throws RemotingException, MQClientException, InterruptedException { + return defaultMQAdminExtImpl.getConsumerRunningInfo(consumerGroup, clientId, jstack); + } + + + @Override + public ConsumeMessageDirectlyResult consumeMessageDirectly(String consumerGroup, String clientId, + String msgId) throws RemotingException, MQClientException, InterruptedException, + MQBrokerException { + return defaultMQAdminExtImpl.consumeMessageDirectly(consumerGroup, clientId, msgId); + } + + + @Override + public List messageTrackDetail(MessageExt msg) throws RemotingException, MQClientException, + InterruptedException, MQBrokerException { + return this.defaultMQAdminExtImpl.messageTrackDetail(msg); + } + + + @Override + public void cloneGroupOffset(String srcGroup, String destGroup, String topic, boolean isOffline) + throws RemotingException, MQClientException, InterruptedException, MQBrokerException { + this.defaultMQAdminExtImpl.cloneGroupOffset(srcGroup, destGroup, topic, isOffline); + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/admin/DefaultMQAdminExtImpl.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/admin/DefaultMQAdminExtImpl.java new file mode 100644 index 000000000..176b4766e --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/admin/DefaultMQAdminExtImpl.java @@ -0,0 +1,840 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.tools.admin; + +import java.io.UnsupportedEncodingException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Properties; +import java.util.Set; + +import org.slf4j.Logger; + +import com.alibaba.rocketmq.client.QueryResult; +import com.alibaba.rocketmq.client.admin.MQAdminExtInner; +import com.alibaba.rocketmq.client.exception.MQBrokerException; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.client.impl.MQClientManager; +import com.alibaba.rocketmq.client.impl.factory.MQClientInstance; +import com.alibaba.rocketmq.client.log.ClientLogger; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.ServiceState; +import com.alibaba.rocketmq.common.TopicConfig; +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.admin.ConsumeStats; +import com.alibaba.rocketmq.common.admin.OffsetWrapper; +import com.alibaba.rocketmq.common.admin.RollbackStats; +import com.alibaba.rocketmq.common.admin.TopicStatsTable; +import com.alibaba.rocketmq.common.help.FAQUrl; +import com.alibaba.rocketmq.common.message.MessageExt; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.common.namesrv.NamesrvUtil; +import com.alibaba.rocketmq.common.protocol.ResponseCode; +import com.alibaba.rocketmq.common.protocol.body.ClusterInfo; +import com.alibaba.rocketmq.common.protocol.body.ConsumeMessageDirectlyResult; +import com.alibaba.rocketmq.common.protocol.body.ConsumerConnection; +import com.alibaba.rocketmq.common.protocol.body.ConsumerRunningInfo; +import com.alibaba.rocketmq.common.protocol.body.GroupList; +import com.alibaba.rocketmq.common.protocol.body.KVTable; +import com.alibaba.rocketmq.common.protocol.body.ProducerConnection; +import com.alibaba.rocketmq.common.protocol.body.QueueTimeSpan; +import com.alibaba.rocketmq.common.protocol.body.TopicList; +import com.alibaba.rocketmq.common.protocol.header.UpdateConsumerOffsetRequestHeader; +import com.alibaba.rocketmq.common.protocol.heartbeat.SubscriptionData; +import com.alibaba.rocketmq.common.protocol.route.BrokerData; +import com.alibaba.rocketmq.common.protocol.route.TopicRouteData; +import com.alibaba.rocketmq.common.subscription.SubscriptionGroupConfig; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.remoting.common.RemotingHelper; +import com.alibaba.rocketmq.remoting.common.RemotingUtil; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; +import com.alibaba.rocketmq.remoting.exception.RemotingConnectException; +import com.alibaba.rocketmq.remoting.exception.RemotingException; +import com.alibaba.rocketmq.remoting.exception.RemotingSendRequestException; +import com.alibaba.rocketmq.remoting.exception.RemotingTimeoutException; +import com.alibaba.rocketmq.tools.admin.api.MessageTrack; +import com.alibaba.rocketmq.tools.admin.api.TrackType; + + +/** + * 所有运维接口都在这里实现 + * + * @author shijia.wxr + * @since 2013-7-21 + */ +public class DefaultMQAdminExtImpl implements MQAdminExt, MQAdminExtInner { + private final Logger log = ClientLogger.getLog(); + private final DefaultMQAdminExt defaultMQAdminExt; + private ServiceState serviceState = ServiceState.CREATE_JUST; + private MQClientInstance mqClientInstance; + private RPCHook rpcHook; + + + public DefaultMQAdminExtImpl(DefaultMQAdminExt defaultMQAdminExt) { + this(defaultMQAdminExt, null); + } + + + public DefaultMQAdminExtImpl(DefaultMQAdminExt defaultMQAdminExt, RPCHook rpcHook) { + this.defaultMQAdminExt = defaultMQAdminExt; + this.rpcHook = rpcHook; + } + + + @Override + public void start() throws MQClientException { + switch (this.serviceState) { + case CREATE_JUST: + this.serviceState = ServiceState.START_FAILED; + + this.defaultMQAdminExt.changeInstanceNameToPID(); + + this.mqClientInstance = + MQClientManager.getInstance().getAndCreateMQClientInstance(this.defaultMQAdminExt, + rpcHook); + + boolean registerOK = + mqClientInstance.registerAdminExt(this.defaultMQAdminExt.getAdminExtGroup(), this); + if (!registerOK) { + this.serviceState = ServiceState.CREATE_JUST; + throw new MQClientException("The adminExt group[" + this.defaultMQAdminExt.getAdminExtGroup() + + "] has created already, specifed another name please."// + + FAQUrl.suggestTodo(FAQUrl.GROUP_NAME_DUPLICATE_URL), null); + } + + mqClientInstance.start(); + + log.info("the adminExt [{}] start OK", this.defaultMQAdminExt.getAdminExtGroup()); + + this.serviceState = ServiceState.RUNNING; + break; + case RUNNING: + case START_FAILED: + case SHUTDOWN_ALREADY: + throw new MQClientException("The AdminExt service state not OK, maybe started once, "// + + this.serviceState// + + FAQUrl.suggestTodo(FAQUrl.CLIENT_SERVICE_NOT_OK), null); + default: + break; + } + } + + + @Override + public void shutdown() { + switch (this.serviceState) { + case CREATE_JUST: + break; + case RUNNING: + this.mqClientInstance.unregisterAdminExt(this.defaultMQAdminExt.getAdminExtGroup()); + this.mqClientInstance.shutdown(); + + log.info("the adminExt [{}] shutdown OK", this.defaultMQAdminExt.getAdminExtGroup()); + this.serviceState = ServiceState.SHUTDOWN_ALREADY; + break; + case SHUTDOWN_ALREADY: + break; + default: + break; + } + } + + + @Override + public void createAndUpdateTopicConfig(String addr, TopicConfig config) throws RemotingException, + MQBrokerException, InterruptedException, MQClientException { + this.mqClientInstance.getMQClientAPIImpl().createTopic(addr, + this.defaultMQAdminExt.getCreateTopicKey(), config, 3000); + } + + + @Override + public void createAndUpdateSubscriptionGroupConfig(String addr, SubscriptionGroupConfig config) + throws RemotingException, MQBrokerException, InterruptedException, MQClientException { + this.mqClientInstance.getMQClientAPIImpl().createSubscriptionGroup(addr, config, 3000); + } + + + @Override + public SubscriptionGroupConfig examineSubscriptionGroupConfig(String addr, String group) { + // TODO Auto-generated method stub + return null; + } + + + @Override + public TopicConfig examineTopicConfig(String addr, String topic) { + // TODO Auto-generated method stub + return null; + } + + + @Override + public TopicStatsTable examineTopicStats(String topic) throws RemotingException, MQClientException, + InterruptedException, MQBrokerException { + TopicRouteData topicRouteData = this.examineTopicRouteInfo(topic); + TopicStatsTable topicStatsTable = new TopicStatsTable(); + + for (BrokerData bd : topicRouteData.getBrokerDatas()) { + String addr = bd.selectBrokerAddr(); + if (addr != null) { + TopicStatsTable tst = + this.mqClientInstance.getMQClientAPIImpl().getTopicStatsInfo(addr, topic, 3000); + topicStatsTable.getOffsetTable().putAll(tst.getOffsetTable()); + } + } + + if (topicStatsTable.getOffsetTable().isEmpty()) { + throw new MQClientException("Not found the topic stats info", null); + } + + return topicStatsTable; + } + + + @Override + public ConsumeStats examineConsumeStats(String consumerGroup) throws RemotingException, + MQClientException, InterruptedException, MQBrokerException { + String retryTopic = MixAll.getRetryTopic(consumerGroup); + TopicRouteData topicRouteData = this.examineTopicRouteInfo(retryTopic); + ConsumeStats result = new ConsumeStats(); + + for (BrokerData bd : topicRouteData.getBrokerDatas()) { + String addr = bd.selectBrokerAddr(); + if (addr != null) { + // 由于查询时间戳会产生IO操作，可能会耗时较长，所以超时时间设置为15s + ConsumeStats consumeStats = + this.mqClientInstance.getMQClientAPIImpl() + .getConsumeStats(addr, consumerGroup, 15000); + result.getOffsetTable().putAll(consumeStats.getOffsetTable()); + long value = result.getConsumeTps() + consumeStats.getConsumeTps(); + result.setConsumeTps(value); + } + } + + if (result.getOffsetTable().isEmpty()) { + throw new MQClientException( + "Not found the consumer group consume stats, because return offset table is empty, maybe the consumer not consume any message", + null); + } + + return result; + } + + + @Override + public ClusterInfo examineBrokerClusterInfo() throws InterruptedException, MQBrokerException, + RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException { + return this.mqClientInstance.getMQClientAPIImpl().getBrokerClusterInfo(3000); + } + + + @Override + public TopicRouteData examineTopicRouteInfo(String topic) throws RemotingException, MQClientException, + InterruptedException { + return this.mqClientInstance.getMQClientAPIImpl().getTopicRouteInfoFromNameServer(topic, 3000); + } + + + @Override + public void putKVConfig(String namespace, String key, String value) { + // TODO Auto-generated method stub + + } + + + @Override + public String getKVConfig(String namespace, String key) throws RemotingException, MQClientException, + InterruptedException { + return this.mqClientInstance.getMQClientAPIImpl().getKVConfigValue(namespace, key, 3000); + } + + + @Override + public void createTopic(String key, String newTopic, int queueNum) throws MQClientException { + createTopic(key, newTopic, queueNum, 0); + } + + + @Override + public void createTopic(String key, String newTopic, int queueNum, int topicSysFlag) + throws MQClientException { + this.mqClientInstance.getMQAdminImpl().createTopic(key, newTopic, queueNum, topicSysFlag); + } + + + @Override + public long searchOffset(MessageQueue mq, long timestamp) throws MQClientException { + return this.mqClientInstance.getMQAdminImpl().searchOffset(mq, timestamp); + } + + + @Override + public long maxOffset(MessageQueue mq) throws MQClientException { + return this.mqClientInstance.getMQAdminImpl().maxOffset(mq); + } + + + @Override + public long minOffset(MessageQueue mq) throws MQClientException { + return this.mqClientInstance.getMQAdminImpl().minOffset(mq); + } + + + @Override + public long earliestMsgStoreTime(MessageQueue mq) throws MQClientException { + return this.mqClientInstance.getMQAdminImpl().earliestMsgStoreTime(mq); + } + + + @Override + public MessageExt viewMessage(String msgId) throws RemotingException, MQBrokerException, + InterruptedException, MQClientException { + return this.mqClientInstance.getMQAdminImpl().viewMessage(msgId); + } + + + @Override + public QueryResult queryMessage(String topic, String key, int maxNum, long begin, long end) + throws MQClientException, InterruptedException { + return this.mqClientInstance.getMQAdminImpl().queryMessage(topic, key, maxNum, begin, end); + } + + + @Override + public ConsumerConnection examineConsumerConnectionInfo(String consumerGroup) + throws InterruptedException, MQBrokerException, RemotingException, MQClientException { + String topic = MixAll.getRetryTopic(consumerGroup); + TopicRouteData topicRouteData = this.examineTopicRouteInfo(topic); + ConsumerConnection result = new ConsumerConnection(); + + for (BrokerData bd : topicRouteData.getBrokerDatas()) { + String addr = bd.selectBrokerAddr(); + if (addr != null) { + return this.mqClientInstance.getMQClientAPIImpl().getConsumerConnectionList(addr, + consumerGroup, 3000); + } + } + + if (result.getConnectionSet().isEmpty()) { + throw new MQClientException("Not found the consumer group connection", null); + } + + return result; + } + + + @Override + public ProducerConnection examineProducerConnectionInfo(String producerGroup, final String topic) + throws RemotingException, MQClientException, InterruptedException, MQBrokerException { + TopicRouteData topicRouteData = this.examineTopicRouteInfo(topic); + ProducerConnection result = new ProducerConnection(); + + for (BrokerData bd : topicRouteData.getBrokerDatas()) { + String addr = bd.selectBrokerAddr(); + if (addr != null) { + return this.mqClientInstance.getMQClientAPIImpl().getProducerConnectionList(addr, + producerGroup, 300); + } + } + + if (result.getConnectionSet().isEmpty()) { + throw new MQClientException("Not found the consumer group connection", null); + } + + return result; + } + + + @Override + public int wipeWritePermOfBroker(final String namesrvAddr, String brokerName) + throws RemotingCommandException, RemotingConnectException, RemotingSendRequestException, + RemotingTimeoutException, InterruptedException, MQClientException { + return this.mqClientInstance.getMQClientAPIImpl() + .wipeWritePermOfBroker(namesrvAddr, brokerName, 3000); + } + + + @Override + public List getNameServerAddressList() { + return this.mqClientInstance.getMQClientAPIImpl().getNameServerAddressList(); + } + + + @Override + public TopicList fetchAllTopicList() throws RemotingException, MQClientException, InterruptedException { + return this.mqClientInstance.getMQClientAPIImpl().getTopicListFromNameServer(3000); + } + + + @Override + public KVTable fetchBrokerRuntimeStats(final String brokerAddr) throws RemotingConnectException, + RemotingSendRequestException, RemotingTimeoutException, InterruptedException, MQBrokerException { + return this.mqClientInstance.getMQClientAPIImpl().getBrokerRuntimeInfo(brokerAddr, 3000); + } + + + @Override + public void deleteTopicInBroker(Set addrs, String topic) throws RemotingException, + MQBrokerException, InterruptedException, MQClientException { + for (String addr : addrs) { + this.mqClientInstance.getMQClientAPIImpl().deleteTopicInBroker(addr, topic, 3000); + } + } + + + @Override + public void deleteTopicInNameServer(Set addrs, String topic) throws RemotingException, + MQBrokerException, InterruptedException, MQClientException { + if (addrs == null) { + String ns = this.mqClientInstance.getMQClientAPIImpl().fetchNameServerAddr(); + addrs = new HashSet(Arrays.asList(ns.split(";"))); + } + for (String addr : addrs) { + this.mqClientInstance.getMQClientAPIImpl().deleteTopicInNameServer(addr, topic, 3000); + } + } + + + @Override + public void deleteSubscriptionGroup(String addr, String groupName) throws RemotingException, + MQBrokerException, InterruptedException, MQClientException { + this.mqClientInstance.getMQClientAPIImpl().deleteSubscriptionGroup(addr, groupName, 3000); + } + + + @Override + public void createAndUpdateKvConfig(String namespace, String key, String value) throws RemotingException, + MQBrokerException, InterruptedException, MQClientException { + this.mqClientInstance.getMQClientAPIImpl().putKVConfigValue(namespace, key, value, 3000); + } + + + @Override + public void deleteKvConfig(String namespace, String key) throws RemotingException, MQBrokerException, + InterruptedException, MQClientException { + this.mqClientInstance.getMQClientAPIImpl().deleteKVConfigValue(namespace, key, 3000); + } + + + @Override + public String getProjectGroupByIp(String ip) throws RemotingException, MQBrokerException, + InterruptedException, MQClientException { + return this.mqClientInstance.getMQClientAPIImpl().getProjectGroupByIp(ip, 3000); + } + + + @Override + public String getIpsByProjectGroup(String projectGroup) throws RemotingException, MQBrokerException, + InterruptedException, MQClientException { + String namespace = NamesrvUtil.NAMESPACE_PROJECT_CONFIG; + return this.mqClientInstance.getMQClientAPIImpl().getKVConfigByValue(namespace, projectGroup, 3000); + } + + + @Override + public void deleteIpsByProjectGroup(String projectGroup) throws RemotingException, MQBrokerException, + InterruptedException, MQClientException { + String namespace = NamesrvUtil.NAMESPACE_PROJECT_CONFIG; + this.mqClientInstance.getMQClientAPIImpl().deleteKVConfigByValue(namespace, projectGroup, 3000); + } + + + @Override + public List resetOffsetByTimestampOld(String consumerGroup, String topic, long timestamp, + boolean force) throws RemotingException, MQBrokerException, InterruptedException, + MQClientException { + TopicRouteData topicRouteData = this.examineTopicRouteInfo(topic); + List rollbackStatsList = new ArrayList(); + for (BrokerData bd : topicRouteData.getBrokerDatas()) { + String addr = bd.selectBrokerAddr(); + if (addr != null) { + // 根据 consumerGroup 查找对应的 mq + ConsumeStats consumeStats = + this.mqClientInstance.getMQClientAPIImpl().getConsumeStats(addr, consumerGroup, 3000); + + // 根据 topic 过滤不需要的 mq + for (Map.Entry entry : consumeStats.getOffsetTable().entrySet()) { + MessageQueue queue = entry.getKey(); + OffsetWrapper offsetWrapper = entry.getValue(); + if (topic.equals(queue.getTopic())) { + // 根据 timestamp 查找对应的offset + long offset = + this.mqClientInstance.getMQClientAPIImpl().searchOffset(addr, topic, + queue.getQueueId(), timestamp, 3000); + // 构建按时间回溯消费进度 + RollbackStats rollbackStats = new RollbackStats(); + rollbackStats.setBrokerName(bd.getBrokerName()); + rollbackStats.setQueueId(queue.getQueueId()); + rollbackStats.setBrokerOffset(offsetWrapper.getBrokerOffset()); + rollbackStats.setConsumerOffset(offsetWrapper.getConsumerOffset()); + rollbackStats.setTimestampOffset(offset); + rollbackStats.setRollbackOffset(offsetWrapper.getConsumerOffset()); + // 更新 offset + if (force || offset <= offsetWrapper.getConsumerOffset()) { + rollbackStats.setRollbackOffset(offset); + UpdateConsumerOffsetRequestHeader requestHeader = + new UpdateConsumerOffsetRequestHeader(); + requestHeader.setConsumerGroup(consumerGroup); + requestHeader.setTopic(topic); + requestHeader.setQueueId(queue.getQueueId()); + requestHeader.setCommitOffset(offset); + this.mqClientInstance.getMQClientAPIImpl().updateConsumerOffset(addr, + requestHeader, 3000); + } + rollbackStatsList.add(rollbackStats); + } + } + } + } + return rollbackStatsList; + } + + + @Override + public KVTable getKVListByNamespace(String namespace) throws RemotingException, MQClientException, + InterruptedException { + return this.mqClientInstance.getMQClientAPIImpl().getKVListByNamespace(namespace, 5000); + } + + + @Override + public void updateBrokerConfig(String brokerAddr, Properties properties) throws RemotingConnectException, + RemotingSendRequestException, RemotingTimeoutException, UnsupportedEncodingException, + InterruptedException, MQBrokerException { + this.mqClientInstance.getMQClientAPIImpl().updateBrokerConfig(brokerAddr, properties, 5000); + } + + + @Override + public Map resetOffsetByTimestamp(String topic, String group, long timestamp, + boolean isForce) throws RemotingException, MQBrokerException, InterruptedException, + MQClientException { + TopicRouteData topicRouteData = this.examineTopicRouteInfo(topic); + List brokerDatas = topicRouteData.getBrokerDatas(); + Map allOffsetTable = new HashMap(); + if (brokerDatas != null) { + for (BrokerData brokerData : brokerDatas) { + String addr = brokerData.selectBrokerAddr(); + if (addr != null) { + Map offsetTable = + this.mqClientInstance.getMQClientAPIImpl().invokeBrokerToResetOffset(addr, topic, + group, timestamp, isForce, 5000); + if (offsetTable != null) { + allOffsetTable.putAll(offsetTable); + } + } + } + } + return allOffsetTable; + } + + + @Override + public Map> getConsumeStatus(String topic, String group, String clientAddr) + throws RemotingException, MQBrokerException, InterruptedException, MQClientException { + TopicRouteData topicRouteData = this.examineTopicRouteInfo(topic); + List brokerDatas = topicRouteData.getBrokerDatas(); + // 每个 broker 上有所有的 consumer 连接，故只需要在一个 broker 执行即可。 + if (brokerDatas != null && brokerDatas.size() > 0) { + String addr = brokerDatas.get(0).selectBrokerAddr(); + if (addr != null) { + return this.mqClientInstance.getMQClientAPIImpl().invokeBrokerToGetConsumerStatus(addr, + topic, group, clientAddr, 5000); + } + } + return Collections.EMPTY_MAP; + } + + + public void createOrUpdateOrderConf(String key, String value, boolean isCluster) + throws RemotingException, MQBrokerException, InterruptedException, MQClientException { + + if (isCluster) { + this.mqClientInstance.getMQClientAPIImpl().putKVConfigValue( + NamesrvUtil.NAMESPACE_ORDER_TOPIC_CONFIG, key, value, 3000); + } + else { + String oldOrderConfs = null; + try { + oldOrderConfs = + this.mqClientInstance.getMQClientAPIImpl().getKVConfigValue( + NamesrvUtil.NAMESPACE_ORDER_TOPIC_CONFIG, key, 3000); + } + catch (Exception e) { + e.printStackTrace(); + } + + // 添加或替换需要更新的 broker + Map orderConfMap = new HashMap(); + if (!UtilAll.isBlank(oldOrderConfs)) { + String[] oldOrderConfArr = oldOrderConfs.split(";"); + for (String oldOrderConf : oldOrderConfArr) { + String[] items = oldOrderConf.split(":"); + orderConfMap.put(items[0], oldOrderConf); + } + } + String[] items = value.split(":"); + orderConfMap.put(items[0], value); + + StringBuilder newOrderConf = new StringBuilder(); + String splitor = ""; + for (String tmp : orderConfMap.keySet()) { + newOrderConf.append(splitor).append(orderConfMap.get(tmp)); + splitor = ";"; + } + this.mqClientInstance.getMQClientAPIImpl().putKVConfigValue( + NamesrvUtil.NAMESPACE_ORDER_TOPIC_CONFIG, key, newOrderConf.toString(), 3000); + } + } + + + @Override + public GroupList queryTopicConsumeByWho(String topic) throws InterruptedException, MQBrokerException, + RemotingException, MQClientException { + TopicRouteData topicRouteData = this.examineTopicRouteInfo(topic); + + for (BrokerData bd : topicRouteData.getBrokerDatas()) { + String addr = bd.selectBrokerAddr(); + if (addr != null) { + return this.mqClientInstance.getMQClientAPIImpl().queryTopicConsumeByWho(addr, topic, 3000); + } + + break; + } + + return null; + } + + + @Override + public Set queryConsumeTimeSpan(final String topic, final String group) + throws InterruptedException, MQBrokerException, RemotingException, MQClientException { + TopicRouteData topicRouteData = this.examineTopicRouteInfo(topic); + for (BrokerData bd : topicRouteData.getBrokerDatas()) { + String addr = bd.selectBrokerAddr(); + if (addr != null) { + return this.mqClientInstance.getMQClientAPIImpl().queryConsumeTimeSpan(addr, topic, group, + 3000); + } + + break; + } + return null; + } + + + @Override + public void resetOffsetNew(String consumerGroup, String topic, long timestamp) throws RemotingException, + MQBrokerException, InterruptedException, MQClientException { + try { + this.resetOffsetByTimestamp(topic, consumerGroup, timestamp, true); + } + catch (MQClientException e) { + if (ResponseCode.CONSUMER_NOT_ONLINE == e.getResponseCode()) { + this.resetOffsetByTimestampOld(consumerGroup, topic, timestamp, true); + return; + } + throw e; + } + } + + + public boolean cleanExpiredConsumerQueueByCluster(ClusterInfo clusterInfo, String cluster) + throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, + MQClientException, InterruptedException { + boolean result = false; + String[] addrs = clusterInfo.retrieveAllAddrByCluster(cluster); + for (String addr : addrs) { + result = cleanExpiredConsumerQueueByAddr(addr); + } + return result; + } + + + @Override + public boolean cleanExpiredConsumerQueue(String cluster) throws RemotingConnectException, + RemotingSendRequestException, RemotingTimeoutException, MQClientException, InterruptedException { + boolean result = false; + try { + ClusterInfo clusterInfo = examineBrokerClusterInfo(); + if (null == cluster || "".equals(cluster)) { + for (String targetCluster : clusterInfo.retrieveAllClusterNames()) { + result = cleanExpiredConsumerQueueByCluster(clusterInfo, targetCluster); + } + } + else { + result = cleanExpiredConsumerQueueByCluster(clusterInfo, cluster); + } + } + catch (MQBrokerException e) { + log.error("cleanExpiredConsumerQueue error.", e); + } + + return result; + } + + + @Override + public boolean cleanExpiredConsumerQueueByAddr(String addr) throws RemotingConnectException, + RemotingSendRequestException, RemotingTimeoutException, MQClientException, InterruptedException { + boolean result = mqClientInstance.getMQClientAPIImpl().cleanExpiredConsumeQueue(addr, 3000L); + log.warn("clean expired ConsumeQueue on target " + addr + " broker " + result); + return result; + } + + + @Override + public ConsumerRunningInfo getConsumerRunningInfo(String consumerGroup, String clientId, boolean jstack) + throws RemotingException, MQClientException, InterruptedException { + String topic = MixAll.RETRY_GROUP_TOPIC_PREFIX + consumerGroup; + TopicRouteData topicRouteData = this.examineTopicRouteInfo(topic); + List brokerDatas = topicRouteData.getBrokerDatas(); + if (brokerDatas != null) { + for (BrokerData brokerData : brokerDatas) { + String addr = brokerData.selectBrokerAddr(); + if (addr != null) { + return this.mqClientInstance.getMQClientAPIImpl().getConsumerRunningInfo(addr, + consumerGroup, clientId, jstack, 12000); + } + } + } + return null; + } + + + @Override + public ConsumeMessageDirectlyResult consumeMessageDirectly(String consumerGroup, String clientId, + String msgId) throws RemotingException, MQClientException, InterruptedException, + MQBrokerException { + MessageExt msg = this.viewMessage(msgId); + + return this.mqClientInstance.getMQClientAPIImpl().consumeMessageDirectly( + RemotingUtil.socketAddress2String(msg.getStoreHost()), consumerGroup, clientId, msgId, 10000); + } + + + public boolean consumed(final MessageExt msg, final String group) throws RemotingException, + MQClientException, InterruptedException, MQBrokerException { + // 查询消费进度 + ConsumeStats cstats = this.examineConsumeStats(group); + + ClusterInfo ci = this.examineBrokerClusterInfo(); + + Iterator> it = cstats.getOffsetTable().entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + MessageQueue mq = next.getKey(); + if (mq.getTopic().equals(msg.getTopic()) && mq.getQueueId() == msg.getQueueId()) { + BrokerData brokerData = ci.getBrokerAddrTable().get(mq.getBrokerName()); + if (brokerData != null) { + String addr = brokerData.getBrokerAddrs().get(MixAll.MASTER_ID); + if (addr.equals(RemotingUtil.socketAddress2String(msg.getStoreHost()))) { + if (next.getValue().getConsumerOffset() > msg.getQueueOffset()) { + return true; + } + } + } + } + } + + return false; + } + + + @Override + public List messageTrackDetail(MessageExt msg) throws RemotingException, MQClientException, + InterruptedException, MQBrokerException { + List result = new ArrayList(); + + GroupList groupList = this.queryTopicConsumeByWho(msg.getTopic()); + + for (String group : groupList.getGroupList()) { + // 查询连接 + MessageTrack mt = new MessageTrack(); + mt.setConsumerGroup(group); + mt.setTrackType(TrackType.UNKNOW_EXCEPTION); + try { + ConsumerConnection cc = this.examineConsumerConnectionInfo(group); + switch (cc.getConsumeType()) { + case CONSUME_ACTIVELY: + mt.setTrackType(TrackType.SUBSCRIBED_BUT_PULL); + break; + case CONSUME_PASSIVELY: + boolean ifConsumed = this.consumed(msg, group); + if (ifConsumed) { + mt.setTrackType(TrackType.SUBSCRIBED_AND_CONSUMED); + + // 查看订阅关系是否匹配 + Iterator> it = + cc.getSubscriptionTable().entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + if (next.getKey().equals(msg.getTopic())) { + if (next.getValue().getTagsSet().contains(msg.getTags()) // + || next.getValue().getTagsSet().contains("*")// + || next.getValue().getTagsSet().isEmpty()// + ) { + + } + else { + mt.setTrackType(TrackType.SUBSCRIBED_BUT_FILTERD); + } + } + } + } + else { + mt.setTrackType(TrackType.SUBSCRIBED_AND_NOT_CONSUME_YET); + } + break; + default: + break; + } + } + catch (Exception e) { + mt.setExceptionDesc(RemotingHelper.exceptionSimpleDesc(e)); + } + + result.add(mt); + } + + return result; + } + + + @Override + public void cloneGroupOffset(String srcGroup, String destGroup, String topic, boolean isOffline) + throws RemotingException, MQClientException, InterruptedException, MQBrokerException { + String retryTopic = MixAll.getRetryTopic(srcGroup); + TopicRouteData topicRouteData = this.examineTopicRouteInfo(retryTopic); + + for (BrokerData bd : topicRouteData.getBrokerDatas()) { + String addr = bd.selectBrokerAddr(); + if (addr != null) { + // 由于查询时间戳会产生IO操作，可能会耗时较长，所以超时时间设置为15s + this.mqClientInstance.getMQClientAPIImpl().cloneGroupOffset(addr, srcGroup, destGroup, topic, + isOffline, 15000); + } + } + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/admin/MQAdminExt.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/admin/MQAdminExt.java new file mode 100644 index 000000000..3b4635e5e --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/admin/MQAdminExt.java @@ -0,0 +1,611 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.tools.admin; + +import java.io.UnsupportedEncodingException; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; + +import com.alibaba.rocketmq.client.MQAdmin; +import com.alibaba.rocketmq.client.exception.MQBrokerException; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.common.TopicConfig; +import com.alibaba.rocketmq.common.admin.ConsumeStats; +import com.alibaba.rocketmq.common.admin.RollbackStats; +import com.alibaba.rocketmq.common.admin.TopicStatsTable; +import com.alibaba.rocketmq.common.message.MessageExt; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.common.protocol.body.ClusterInfo; +import com.alibaba.rocketmq.common.protocol.body.ConsumeMessageDirectlyResult; +import com.alibaba.rocketmq.common.protocol.body.ConsumerConnection; +import com.alibaba.rocketmq.common.protocol.body.ConsumerRunningInfo; +import com.alibaba.rocketmq.common.protocol.body.GroupList; +import com.alibaba.rocketmq.common.protocol.body.KVTable; +import com.alibaba.rocketmq.common.protocol.body.ProducerConnection; +import com.alibaba.rocketmq.common.protocol.body.QueueTimeSpan; +import com.alibaba.rocketmq.common.protocol.body.TopicList; +import com.alibaba.rocketmq.common.protocol.route.TopicRouteData; +import com.alibaba.rocketmq.common.subscription.SubscriptionGroupConfig; +import com.alibaba.rocketmq.remoting.exception.RemotingCommandException; +import com.alibaba.rocketmq.remoting.exception.RemotingConnectException; +import com.alibaba.rocketmq.remoting.exception.RemotingException; +import com.alibaba.rocketmq.remoting.exception.RemotingSendRequestException; +import com.alibaba.rocketmq.remoting.exception.RemotingTimeoutException; +import com.alibaba.rocketmq.tools.admin.api.MessageTrack; + + +/** + * MQ管理类接口，涉及所有与MQ管理相关的对外接口
+ * 包括Topic创建、订阅组创建、配置修改等 + * + * @since 2013-7-14 + */ +public interface MQAdminExt extends MQAdmin { + public void start() throws MQClientException; + + + public void shutdown(); + + + /** + * 更新Broker配置 + * + * @param brokerAddr + * @param properties + * @throws MQBrokerException + * @throws InterruptedException + * @throws UnsupportedEncodingException + * @throws RemotingTimeoutException + * @throws RemotingSendRequestException + * @throws RemotingConnectException + */ + public void updateBrokerConfig(final String brokerAddr, final Properties properties) + throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, + UnsupportedEncodingException, InterruptedException, MQBrokerException; + + + /** + * 向指定Broker创建或者更新Topic配置 + * + * @param addr + * @param config + * @throws MQClientException + * @throws InterruptedException + * @throws MQBrokerException + * @throws RemotingException + */ + public void createAndUpdateTopicConfig(final String addr, final TopicConfig config) + throws RemotingException, MQBrokerException, InterruptedException, MQClientException; + + + /** + * 向指定Broker创建或者更新订阅组配置 + * + * @param addr + * @param config + * @throws MQClientException + * @throws InterruptedException + * @throws MQBrokerException + * @throws RemotingException + */ + public void createAndUpdateSubscriptionGroupConfig(final String addr, final SubscriptionGroupConfig config) + throws RemotingException, MQBrokerException, InterruptedException, MQClientException; + + + /** + * 查询指定Broker的订阅组配置 + * + * @param addr + * @param group + * @return + */ + public SubscriptionGroupConfig examineSubscriptionGroupConfig(final String addr, final String group); + + + /** + * 查询指定Broker的Topic配置 + * + * @param addr + * @param topic + * @return + */ + public TopicConfig examineTopicConfig(final String addr, final String topic); + + + /** + * 查询Topic Offset信息 + * + * @param topic + * @return + */ + public TopicStatsTable examineTopicStats(final String topic) throws RemotingException, MQClientException, + InterruptedException, MQBrokerException; + + + /** + * 从Name Server获取所有Topic列表 + * + * @return + * @throws InterruptedException + * @throws MQClientException + * @throws RemotingException + */ + public TopicList fetchAllTopicList() throws RemotingException, MQClientException, InterruptedException; + + + /** + * 获取Broker运行时数据 + * + * @return + * @throws MQBrokerException + * @throws InterruptedException + * @throws RemotingTimeoutException + * @throws RemotingSendRequestException + * @throws RemotingConnectException + */ + public KVTable fetchBrokerRuntimeStats(final String brokerAddr) throws RemotingConnectException, + RemotingSendRequestException, RemotingTimeoutException, InterruptedException, MQBrokerException; + + + /** + * 查询消费进度 + * + * @param consumerGroup + * @return + * @throws InterruptedException + * @throws MQClientException + * @throws RemotingException + * @throws MQBrokerException + */ + public ConsumeStats examineConsumeStats(final String consumerGroup) throws RemotingException, + MQClientException, InterruptedException, MQBrokerException; + + + /** + * 查看集群信息 + * + * @return + */ + public ClusterInfo examineBrokerClusterInfo() throws InterruptedException, MQBrokerException, + RemotingTimeoutException, RemotingSendRequestException, RemotingConnectException; + + + /** + * 查看Topic路由信息 + * + * @param topic + * @return + */ + public TopicRouteData examineTopicRouteInfo(final String topic) throws RemotingException, + MQClientException, InterruptedException; + + + /** + * 查看Consumer网络连接、订阅关系 + * + * @param consumerGroup + * @return + * @throws MQBrokerException + * @throws InterruptedException + * @throws RemotingTimeoutException + * @throws RemotingSendRequestException + * @throws RemotingConnectException + * @throws MQClientException + * @throws RemotingException + */ + public ConsumerConnection examineConsumerConnectionInfo(final String consumerGroup) + throws RemotingConnectException, RemotingSendRequestException, RemotingTimeoutException, + InterruptedException, MQBrokerException, RemotingException, MQClientException; + + + /** + * 查看Producer网络连接 + * + * @param producerGroup + * @param topic + * @return + * @throws InterruptedException + * @throws MQClientException + * @throws RemotingException + * @throws MQBrokerException + */ + public ProducerConnection examineProducerConnectionInfo(final String producerGroup, final String topic) + throws RemotingException, MQClientException, InterruptedException, MQBrokerException; + + + /** + * 获取Name Server地址列表 + * + * @return + */ + public List getNameServerAddressList(); + + + /** + * 清除某个Broker的写权限，针对所有Name Server + * + * @param brokerName + * @return 返回清除了多少个topic + * @throws MQClientException + * @throws InterruptedException + * @throws RemotingTimeoutException + * @throws RemotingSendRequestException + * @throws RemotingConnectException + * @throws RemotingCommandException + */ + public int wipeWritePermOfBroker(final String namesrvAddr, String brokerName) + throws RemotingCommandException, RemotingConnectException, RemotingSendRequestException, + RemotingTimeoutException, InterruptedException, MQClientException; + + + /** + * 向Name Server增加一个配置项 + * + * @param namespace + * @param key + * @param value + */ + public void putKVConfig(final String namespace, final String key, final String value); + + + /** + * 从Name Server获取一个配置项 + * + * @param namespace + * @param key + * @return + */ + public String getKVConfig(final String namespace, final String key) throws RemotingException, + MQClientException, InterruptedException; + + + /** + * 获取指定Namespace下的所有kv + * + * @param namespace + * @return + * @throws InterruptedException + * @throws MQClientException + * @throws RemotingException + */ + public KVTable getKVListByNamespace(final String namespace) throws RemotingException, MQClientException, + InterruptedException; + + + /** + * 删除 broker 上的 topic 信息 + * + * @param addrs + * @param topic + * @throws RemotingException + * @throws MQBrokerException + * @throws InterruptedException + * @throws MQClientException + */ + public void deleteTopicInBroker(final Set addrs, final String topic) throws RemotingException, + MQBrokerException, InterruptedException, MQClientException; + + + /** + * 删除 broker 上的 topic 信息 + * + * @param addrs + * @param topic + * @throws RemotingException + * @throws MQBrokerException + * @throws InterruptedException + * @throws MQClientException + */ + public void deleteTopicInNameServer(final Set addrs, final String topic) + throws RemotingException, MQBrokerException, InterruptedException, MQClientException; + + + /** + * 删除 broker 上的 subscription group 信息 + * + * @param addr + * @param groupName + * @throws RemotingException + * @throws MQBrokerException + * @throws InterruptedException + * @throws MQClientException + */ + public void deleteSubscriptionGroup(final String addr, String groupName) throws RemotingException, + MQBrokerException, InterruptedException, MQClientException; + + + /** + * 在 namespace 上添加或者更新 KV 配置 + * + * @param namespace + * @param key + * @param value + * @throws RemotingException + * @throws MQBrokerException + * @throws InterruptedException + * @throws MQClientException + */ + public void createAndUpdateKvConfig(String namespace, String key, String value) throws RemotingException, + MQBrokerException, InterruptedException, MQClientException; + + + /** + * 删除 namespace 上的 KV 配置 + * + * @param namespace + * @param key + * @throws RemotingException + * @throws MQBrokerException + * @throws InterruptedException + * @throws MQClientException + */ + public void deleteKvConfig(String namespace, String key) throws RemotingException, MQBrokerException, + InterruptedException, MQClientException; + + + /** + * 通过 server ip 获取 project 信息 + * + * @param ip + * @throws RemotingException + * @throws MQBrokerException + * @throws InterruptedException + * @throws MQClientException + * @return + */ + public String getProjectGroupByIp(String ip) throws RemotingException, MQBrokerException, + InterruptedException, MQClientException; + + + /** + * 通过 project 获取所有的 server ip 信息 + * + * @param projectGroup + * @throws RemotingException + * @throws MQBrokerException + * @throws InterruptedException + * @throws MQClientException + * @return + */ + public String getIpsByProjectGroup(String projectGroup) throws RemotingException, MQBrokerException, + InterruptedException, MQClientException; + + + /** + * 删除 project group 对应的所有 server ip + * + * @param key + * @throws RemotingException + * @throws MQBrokerException + * @throws InterruptedException + * @throws MQClientException + */ + public void deleteIpsByProjectGroup(String key) throws RemotingException, MQBrokerException, + InterruptedException, MQClientException; + + + /** + * 按照时间回溯消费进度(客户端需要重启) + * + * @param consumerGroup + * @param topic + * @param timestamp + * @param force + * @throws RemotingException + * @throws MQBrokerException + * @throws InterruptedException + * @throws MQClientException + * @return + */ + public List resetOffsetByTimestampOld(String consumerGroup, String topic, long timestamp, + boolean force) throws RemotingException, MQBrokerException, InterruptedException, + MQClientException; + + + /** + * 按照时间回溯消费进度(客户端不需要重启) + * + * @param topic + * @param group + * @param timestamp + * @param isForce + * @throws RemotingException + * @throws MQBrokerException + * @throws InterruptedException + * @throws MQClientException + * @return + */ + public Map resetOffsetByTimestamp(String topic, String group, long timestamp, + boolean isForce) throws RemotingException, MQBrokerException, InterruptedException, + MQClientException; + + + /** + * 重置消费进度，无论Consumer是否在线，都可以执行。不保证最终结果是否成功，需要调用方通过消费进度查询来再次确认 + * + * @param consumerGroup + * @param topic + * @param timestamp + * @throws InterruptedException + * @throws MQBrokerException + * @throws RemotingException + * @throws MQClientException + */ + public void resetOffsetNew(String consumerGroup, String topic, long timestamp) throws RemotingException, + MQBrokerException, InterruptedException, MQClientException; + + + /** + * 通过客户端查看消费者的消费情况 + * + * @param topic + * @param group + * @param clientAddr + * @return + * @throws RemotingException + * @throws MQBrokerException + * @throws InterruptedException + * @throws MQClientException + */ + public Map> getConsumeStatus(String topic, String group, String clientAddr) + throws RemotingException, MQBrokerException, InterruptedException, MQClientException; + + + /** + * 创建或更新顺序消息的分区配置 + * + * @param key + * @param value + * @param isCluster + * @throws RemotingException + * @throws MQBrokerException + * @throws InterruptedException + * @throws MQClientException + */ + public void createOrUpdateOrderConf(String key, String value, boolean isCluster) + throws RemotingException, MQBrokerException, InterruptedException, MQClientException; + + + /** + * 根据Topic查询被哪些订阅组消费 + * + * @param topic + * @return + * @throws MQBrokerException + * @throws InterruptedException + * @throws RemotingTimeoutException + * @throws RemotingSendRequestException + * @throws RemotingConnectException + * @throws MQClientException + * @throws RemotingException + */ + public GroupList queryTopicConsumeByWho(final String topic) throws RemotingConnectException, + RemotingSendRequestException, RemotingTimeoutException, InterruptedException, MQBrokerException, + RemotingException, MQClientException; + + + /** + * 根据 topic 和 group 获取消息的时间跨度 + * + * @param topic + * @param group + * @return + * @throws RemotingConnectException + * @throws RemotingSendRequestException + * @throws RemotingTimeoutException + * @throws InterruptedException + * @throws MQBrokerException + * @throws RemotingException + * @throws MQClientException + */ + public Set queryConsumeTimeSpan(final String topic, final String group) + throws InterruptedException, MQBrokerException, RemotingException, MQClientException; + + + /** + * 触发清理失效的消费队列 + * + * @param cluster + * null则表示所有集群 + * @return 清理是否成功 + * @throws RemotingConnectException + * @throws RemotingSendRequestException + * @throws RemotingTimeoutException + * @throws MQClientException + * @throws InterruptedException + */ + public boolean cleanExpiredConsumerQueue(String cluster) throws RemotingConnectException, + RemotingSendRequestException, RemotingTimeoutException, MQClientException, InterruptedException; + + + /** + * 触发指定的broker清理失效的消费队列 + * + * @param addr + * @return 清理是否成功 + * @throws RemotingConnectException + * @throws RemotingSendRequestException + * @throws RemotingTimeoutException + * @throws MQClientException + * @throws InterruptedException + */ + public boolean cleanExpiredConsumerQueueByAddr(String addr) throws RemotingConnectException, + RemotingSendRequestException, RemotingTimeoutException, MQClientException, InterruptedException; + + + /** + * 查询Consumer内存数据结构 + * + * @param consumerGroup + * @param clientId + * @return + * @throws InterruptedException + * @throws MQClientException + * @throws RemotingException + */ + public ConsumerRunningInfo getConsumerRunningInfo(final String consumerGroup, final String clientId, + final boolean jstack) throws RemotingException, MQClientException, InterruptedException; + + + /** + * 向指定Consumer发送某条消息 + * + * @param consumerGroup + * @param clientId + * @param msgId + * @return + * @throws InterruptedException + * @throws MQClientException + * @throws RemotingException + * @throws MQBrokerException + */ + public ConsumeMessageDirectlyResult consumeMessageDirectly(String consumerGroup, // + String clientId, // + String msgId) throws RemotingException, MQClientException, InterruptedException, + MQBrokerException; + + + /** + * 查询消息被谁消费了 + * + * @param msg + * @return + * @throws RemotingException + * @throws MQClientException + * @throws InterruptedException + * @throws MQBrokerException + */ + public List messageTrackDetail(MessageExt msg) throws RemotingException, MQClientException, + InterruptedException, MQBrokerException; + + + /** + * 克隆某一个组的消费进度到新的组 + * + * @param srcGroup + * @param destGroup + * @param topic + * @param isOffline + * @throws RemotingException + * @throws MQClientException + * @throws InterruptedException + * @throws MQBrokerException + */ + public void cloneGroupOffset(String srcGroup, String destGroup, String topic, boolean isOffline) + throws RemotingException, MQClientException, InterruptedException, MQBrokerException; +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/admin/api/MessageTrack.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/admin/api/MessageTrack.java new file mode 100644 index 000000000..a80831585 --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/admin/api/MessageTrack.java @@ -0,0 +1,44 @@ +package com.alibaba.rocketmq.tools.admin.api; + +public class MessageTrack { + private String consumerGroup; + private TrackType trackType; + private String exceptionDesc; + + + public String getConsumerGroup() { + return consumerGroup; + } + + + public void setConsumerGroup(String consumerGroup) { + this.consumerGroup = consumerGroup; + } + + + public TrackType getTrackType() { + return trackType; + } + + + public void setTrackType(TrackType trackType) { + this.trackType = trackType; + } + + + public String getExceptionDesc() { + return exceptionDesc; + } + + + public void setExceptionDesc(String exceptionDesc) { + this.exceptionDesc = exceptionDesc; + } + + + @Override + public String toString() { + return "MessageTrack [consumerGroup=" + consumerGroup + ", trackType=" + trackType + + ", exceptionDesc=" + exceptionDesc + "]"; + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/admin/api/TrackType.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/admin/api/TrackType.java new file mode 100644 index 000000000..d4af63391 --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/admin/api/TrackType.java @@ -0,0 +1,14 @@ +package com.alibaba.rocketmq.tools.admin.api; + +public enum TrackType { + // 订阅了，而且消费了（Offset越过了） + SUBSCRIBED_AND_CONSUMED, + // 订阅了，但是被过滤掉了 + SUBSCRIBED_BUT_FILTERD, + // 订阅了，但是是PULL，结果未知 + SUBSCRIBED_BUT_PULL, + // 订阅了，但是没有消费（Offset小） + SUBSCRIBED_AND_NOT_CONSUME_YET, + // 未知异常 + UNKNOW_EXCEPTION, +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/CommandUtil.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/CommandUtil.java new file mode 100644 index 000000000..40530fab9 --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/CommandUtil.java @@ -0,0 +1,98 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.tools.command; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; + +import com.alibaba.rocketmq.client.exception.MQBrokerException; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.protocol.body.ClusterInfo; +import com.alibaba.rocketmq.common.protocol.route.BrokerData; +import com.alibaba.rocketmq.remoting.exception.RemotingConnectException; +import com.alibaba.rocketmq.remoting.exception.RemotingSendRequestException; +import com.alibaba.rocketmq.remoting.exception.RemotingTimeoutException; +import com.alibaba.rocketmq.tools.admin.MQAdminExt; + + +/** + * 各个子命令的接口 + * + * @author shijia.wxr + * @since 2013-7-25 + */ +public class CommandUtil { + public static Set fetchMasterAddrByClusterName(final MQAdminExt adminExt, final String clusterName) + throws InterruptedException, RemotingConnectException, RemotingTimeoutException, + RemotingSendRequestException, MQBrokerException { + Set masterSet = new HashSet(); + + ClusterInfo clusterInfoSerializeWrapper = adminExt.examineBrokerClusterInfo(); + + Set brokerNameSet = clusterInfoSerializeWrapper.getClusterAddrTable().get(clusterName); + + if (brokerNameSet != null) { + for (String brokerName : brokerNameSet) { + BrokerData brokerData = clusterInfoSerializeWrapper.getBrokerAddrTable().get(brokerName); + if (brokerData != null) { + + String addr = brokerData.getBrokerAddrs().get(MixAll.MASTER_ID); + if (addr != null) { + masterSet.add(addr); + } + } + } + } + else { + System.out + .printf("[error] Make sure the specified clusterName exists or the nameserver which connected is correct."); + } + + return masterSet; + } + + + public static Set fetchBrokerNameByClusterName(final MQAdminExt adminExt, final String clusterName) + throws Exception { + ClusterInfo clusterInfoSerializeWrapper = adminExt.examineBrokerClusterInfo(); + Set brokerNameSet = clusterInfoSerializeWrapper.getClusterAddrTable().get(clusterName); + if (brokerNameSet.isEmpty()) { + throw new Exception( + "Make sure the specified clusterName exists or the nameserver which connected is correct."); + } + return brokerNameSet; + } + + + public static String fetchBrokerNameByAddr(final MQAdminExt adminExt, final String addr) throws Exception { + ClusterInfo clusterInfoSerializeWrapper = adminExt.examineBrokerClusterInfo(); + HashMap brokerAddrTable = + clusterInfoSerializeWrapper.getBrokerAddrTable(); + Iterator> it = brokerAddrTable.entrySet().iterator(); + while (it.hasNext()) { + Map.Entry entry = it.next(); + HashMap brokerAddrs = entry.getValue().getBrokerAddrs(); + if (brokerAddrs.containsValue(addr)) + return entry.getKey(); + } + throw new Exception( + "Make sure the specified broker addr exists or the nameserver which connected is correct."); + } + +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/MQAdminStartup.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/MQAdminStartup.java new file mode 100644 index 000000000..6e60720d2 --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/MQAdminStartup.java @@ -0,0 +1,227 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.tools.command; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.PosixParser; +import org.slf4j.LoggerFactory; + +import ch.qos.logback.classic.LoggerContext; +import ch.qos.logback.classic.joran.JoranConfigurator; +import ch.qos.logback.core.joran.spi.JoranException; + +import com.alibaba.rocketmq.common.MQVersion; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.remoting.protocol.RemotingCommand; +import com.alibaba.rocketmq.srvutil.ServerUtil; +import com.alibaba.rocketmq.tools.command.broker.BrokerStatusSubCommand; +import com.alibaba.rocketmq.tools.command.broker.CleanExpiredCQSubCommand; +import com.alibaba.rocketmq.tools.command.broker.UpdateBrokerConfigSubCommand; +import com.alibaba.rocketmq.tools.command.cluster.ClusterListSubCommand; +import com.alibaba.rocketmq.tools.command.connection.ConsumerConnectionSubCommand; +import com.alibaba.rocketmq.tools.command.connection.ProducerConnectionSubCommand; +import com.alibaba.rocketmq.tools.command.consumer.ConsumerProgressSubCommand; +import com.alibaba.rocketmq.tools.command.consumer.ConsumerStatusSubCommand; +import com.alibaba.rocketmq.tools.command.consumer.DeleteSubscriptionGroupCommand; +import com.alibaba.rocketmq.tools.command.consumer.StartMonitoringSubCommand; +import com.alibaba.rocketmq.tools.command.consumer.UpdateSubGroupSubCommand; +import com.alibaba.rocketmq.tools.command.message.CheckMsgSubCommand; +import com.alibaba.rocketmq.tools.command.message.PrintMessageSubCommand; +import com.alibaba.rocketmq.tools.command.message.QueryMsgByIdSubCommand; +import com.alibaba.rocketmq.tools.command.message.QueryMsgByKeySubCommand; +import com.alibaba.rocketmq.tools.command.message.QueryMsgByOffsetSubCommand; +import com.alibaba.rocketmq.tools.command.namesrv.DeleteKvConfigCommand; +import com.alibaba.rocketmq.tools.command.namesrv.DeleteProjectGroupCommand; +import com.alibaba.rocketmq.tools.command.namesrv.GetProjectGroupCommand; +import com.alibaba.rocketmq.tools.command.namesrv.UpdateKvConfigCommand; +import com.alibaba.rocketmq.tools.command.namesrv.UpdateProjectGroupCommand; +import com.alibaba.rocketmq.tools.command.namesrv.WipeWritePermSubCommand; +import com.alibaba.rocketmq.tools.command.offset.CloneGroupOffsetCommand; +import com.alibaba.rocketmq.tools.command.offset.ResetOffsetByTimeCommand; +import com.alibaba.rocketmq.tools.command.topic.DeleteTopicSubCommand; +import com.alibaba.rocketmq.tools.command.topic.TopicListSubCommand; +import com.alibaba.rocketmq.tools.command.topic.TopicRouteSubCommand; +import com.alibaba.rocketmq.tools.command.topic.TopicStatusSubCommand; +import com.alibaba.rocketmq.tools.command.topic.UpdateOrderConfCommand; +import com.alibaba.rocketmq.tools.command.topic.UpdateTopicSubCommand; + + +/** + * mqadmin启动程序 + * + * @author shijia.wxr + * @since 2013-7-25 + */ +public class MQAdminStartup { + protected static List subCommandList = new ArrayList(); + static { + subCommandList.add(new UpdateTopicSubCommand()); + subCommandList.add(new DeleteTopicSubCommand()); + subCommandList.add(new UpdateSubGroupSubCommand()); + subCommandList.add(new DeleteSubscriptionGroupCommand()); + subCommandList.add(new UpdateBrokerConfigSubCommand()); + + subCommandList.add(new TopicRouteSubCommand()); + subCommandList.add(new TopicStatusSubCommand()); + + subCommandList.add(new BrokerStatusSubCommand()); + subCommandList.add(new QueryMsgByIdSubCommand()); + subCommandList.add(new QueryMsgByKeySubCommand()); + subCommandList.add(new QueryMsgByOffsetSubCommand()); + subCommandList.add(new PrintMessageSubCommand()); + + subCommandList.add(new ProducerConnectionSubCommand()); + subCommandList.add(new ConsumerConnectionSubCommand()); + subCommandList.add(new ConsumerProgressSubCommand()); + subCommandList.add(new ConsumerStatusSubCommand()); + subCommandList.add(new CloneGroupOffsetCommand()); + + subCommandList.add(new ClusterListSubCommand()); + subCommandList.add(new TopicListSubCommand()); + + subCommandList.add(new UpdateKvConfigCommand()); + subCommandList.add(new DeleteKvConfigCommand()); + + subCommandList.add(new UpdateProjectGroupCommand()); + subCommandList.add(new DeleteProjectGroupCommand()); + subCommandList.add(new GetProjectGroupCommand()); + subCommandList.add(new WipeWritePermSubCommand()); + subCommandList.add(new ResetOffsetByTimeCommand()); + + subCommandList.add(new UpdateOrderConfCommand()); + subCommandList.add(new CleanExpiredCQSubCommand()); + + subCommandList.add(new StartMonitoringSubCommand()); + subCommandList.add(new CheckMsgSubCommand()); + } + + + public static void main(String[] args) { + main0(args, null); + } + + + public static void main0(String[] args, RPCHook rpcHook) { + System.setProperty(RemotingCommand.RemotingVersionKey, Integer.toString(MQVersion.CurrentVersion)); + + try { + initLogback(); + switch (args.length) { + case 0: + printHelp(); + break; + case 2: + if (args[0].equals("help")) { + SubCommand cmd = findSubCommand(args[1]); + if (cmd != null) { + Options options = ServerUtil.buildCommandlineOptions(new Options()); + options = cmd.buildCommandlineOptions(options); + if (options != null) { + ServerUtil.printCommandLineHelp("mqadmin " + cmd.commandName(), options); + } + } + else { + System.out.println("The sub command \'" + args[1] + "\' not exist."); + } + break; + } + case 1: + default: + SubCommand cmd = findSubCommand(args[0]); + if (cmd != null) { + // 将main中的args转化为子命令的args（去除第一个参数） + String[] subargs = parseSubArgs(args); + + // 解析命令行 + Options options = ServerUtil.buildCommandlineOptions(new Options()); + final CommandLine commandLine = + ServerUtil.parseCmdLine("mqadmin " + cmd.commandName(), subargs, + cmd.buildCommandlineOptions(options), new PosixParser()); + if (null == commandLine) { + System.exit(-1); + return; + } + + if (commandLine.hasOption('n')) { + String namesrvAddr = commandLine.getOptionValue('n'); + System.setProperty(MixAll.NAMESRV_ADDR_PROPERTY, namesrvAddr); + } + + cmd.execute(commandLine, options, rpcHook); + } + else { + System.out.println("The sub command \'" + args[0] + "\' not exist."); + } + break; + } + } + catch (Exception e) { + e.printStackTrace(); + } + } + + + private static void initLogback() throws JoranException { + String rocketmqHome = + System.getProperty(MixAll.ROCKETMQ_HOME_PROPERTY, System.getenv(MixAll.ROCKETMQ_HOME_ENV)); + + // 初始化Logback + LoggerContext lc = (LoggerContext) LoggerFactory.getILoggerFactory(); + JoranConfigurator configurator = new JoranConfigurator(); + configurator.setContext(lc); + lc.reset(); + configurator.doConfigure(rocketmqHome + "/conf/logback_tools.xml"); + } + + + private static String[] parseSubArgs(String[] args) { + if (args.length > 1) { + String[] result = new String[args.length - 1]; + for (int i = 0; i < args.length - 1; i++) { + result[i] = args[i + 1]; + } + return result; + } + return null; + } + + + private static SubCommand findSubCommand(final String name) { + for (SubCommand cmd : subCommandList) { + if (cmd.commandName().toUpperCase().equals(name.toUpperCase())) { + return cmd; + } + } + + return null; + } + + + private static void printHelp() { + System.out.println("The most commonly used mqadmin commands are:"); + + for (SubCommand cmd : subCommandList) { + System.out.printf(" %-20s %s\n", cmd.commandName(), cmd.commandDesc()); + } + + System.out.println("\nSee 'mqadmin help ' for more information on a specific command."); + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/SubCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/SubCommand.java new file mode 100644 index 000000000..bd794c558 --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/SubCommand.java @@ -0,0 +1,41 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.tools.command; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Options; + +import com.alibaba.rocketmq.remoting.RPCHook; + + +/** + * 各个子命令的接口 + * + * @author shijia.wxr + * @since 2013-7-25 + */ +public interface SubCommand { + public String commandName(); + + + public String commandDesc(); + + + public Options buildCommandlineOptions(final Options options); + + + public void execute(final CommandLine commandLine, final Options options, RPCHook rpcHook); +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/broker/BrokerStatusSubCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/broker/BrokerStatusSubCommand.java new file mode 100644 index 000000000..fb0fc2c4b --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/broker/BrokerStatusSubCommand.java @@ -0,0 +1,92 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.tools.command.broker; + +import java.util.Iterator; +import java.util.Map.Entry; +import java.util.TreeMap; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; + +import com.alibaba.rocketmq.common.protocol.body.KVTable; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.tools.admin.DefaultMQAdminExt; +import com.alibaba.rocketmq.tools.command.SubCommand; + + +/** + * 获取Broker运行时统计信息 + * + * @author shijia.wxr + * @since 2013-8-14 + */ +public class BrokerStatusSubCommand implements SubCommand { + + @Override + public String commandName() { + return "brokerStatus"; + } + + + @Override + public String commandDesc() { + return "Fetch broker runtime status data"; + } + + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("b", "brokerAddr", true, "Broker address"); + opt.setRequired(true); + options.addOption(opt); + + return options; + } + + + @Override + public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) { + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + + try { + defaultMQAdminExt.start(); + + String brokerAddr = commandLine.getOptionValue('b').trim(); + + KVTable kvTable = defaultMQAdminExt.fetchBrokerRuntimeStats(brokerAddr); + + // 为了排序 + TreeMap tmp = new TreeMap(); + tmp.putAll(kvTable.getTable()); + + Iterator> it = tmp.entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + System.out.printf("%-32s: %s\n", next.getKey(), next.getValue()); + } + } + catch (Exception e) { + e.printStackTrace(); + } + finally { + defaultMQAdminExt.shutdown(); + } + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/broker/CleanExpiredCQSubCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/broker/CleanExpiredCQSubCommand.java new file mode 100644 index 000000000..80cefbccd --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/broker/CleanExpiredCQSubCommand.java @@ -0,0 +1,71 @@ +package com.alibaba.rocketmq.tools.command.broker; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; + +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.tools.admin.DefaultMQAdminExt; +import com.alibaba.rocketmq.tools.command.SubCommand; + + +/** + * @auther lansheng.zj + */ +public class CleanExpiredCQSubCommand implements SubCommand { + + @Override + public String commandName() { + return "cleanExpiredCQ"; + } + + + @Override + public String commandDesc() { + return "Clean expired ConsumeQueue on broker."; + } + + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("b", "brokerAddr", true, "Broker address"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("c", "cluster", true, "clustername"); + opt.setRequired(false); + options.addOption(opt); + + return options; + } + + + @Override + public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) { + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + + try { + boolean result = false; + defaultMQAdminExt.start(); + if (commandLine.hasOption('b')) { + String addr = commandLine.getOptionValue('b').trim(); + result = defaultMQAdminExt.cleanExpiredConsumerQueueByAddr(addr); + + } + else { + String cluster = commandLine.getOptionValue('c'); + if (null != cluster) + cluster = cluster.trim(); + result = defaultMQAdminExt.cleanExpiredConsumerQueue(cluster); + } + System.out.println(result ? "success" : "false"); + } + catch (Exception e) { + e.printStackTrace(); + } + finally { + defaultMQAdminExt.shutdown(); + } + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/broker/UpdateBrokerConfigSubCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/broker/UpdateBrokerConfigSubCommand.java new file mode 100644 index 000000000..ee75c8668 --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/broker/UpdateBrokerConfigSubCommand.java @@ -0,0 +1,119 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.tools.command.broker; + +import java.util.Properties; +import java.util.Set; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; + +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.srvutil.ServerUtil; +import com.alibaba.rocketmq.tools.admin.DefaultMQAdminExt; +import com.alibaba.rocketmq.tools.command.CommandUtil; +import com.alibaba.rocketmq.tools.command.SubCommand; + + +/** + * 更新Broker配置文件 + * + * @author shijia.wxr + * @since 2013-10-20 + */ +public class UpdateBrokerConfigSubCommand implements SubCommand { + + @Override + public String commandName() { + return "updateBrokerConfig"; + } + + + @Override + public String commandDesc() { + return "Update broker's config"; + } + + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("b", "brokerAddr", true, "update which broker"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("c", "clusterName", true, "update which cluster"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("k", "key", true, "config key"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("v", "value", true, "config value"); + opt.setRequired(true); + options.addOption(opt); + + return options; + } + + + @Override + public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) { + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + + try { + String key = commandLine.getOptionValue('k').trim(); + String value = commandLine.getOptionValue('v').trim(); + Properties properties = new Properties(); + properties.put(key, value); + + if (commandLine.hasOption('b')) { + String brokerAddr = commandLine.getOptionValue('b').trim(); + + defaultMQAdminExt.start(); + + defaultMQAdminExt.updateBrokerConfig(brokerAddr, properties); + System.out.printf("update broker config success, %s\n", brokerAddr); + return; + + } + else if (commandLine.hasOption('c')) { + String clusterName = commandLine.getOptionValue('c').trim(); + + defaultMQAdminExt.start(); + + Set masterSet = + CommandUtil.fetchMasterAddrByClusterName(defaultMQAdminExt, clusterName); + for (String brokerAddr : masterSet) { + defaultMQAdminExt.updateBrokerConfig(brokerAddr, properties); + System.out.printf("update broker config success, %s\n", brokerAddr); + } + return; + } + + ServerUtil.printCommandLineHelp("mqadmin " + this.commandName(), options); + } + catch (Exception e) { + e.printStackTrace(); + } + finally { + defaultMQAdminExt.shutdown(); + } + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/cluster/ClusterListSubCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/cluster/ClusterListSubCommand.java new file mode 100644 index 000000000..1100087d1 --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/cluster/ClusterListSubCommand.java @@ -0,0 +1,256 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.tools.command.cluster; + +import java.util.Iterator; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; + +import com.alibaba.rocketmq.client.exception.MQBrokerException; +import com.alibaba.rocketmq.common.protocol.body.ClusterInfo; +import com.alibaba.rocketmq.common.protocol.body.KVTable; +import com.alibaba.rocketmq.common.protocol.route.BrokerData; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.remoting.exception.RemotingConnectException; +import com.alibaba.rocketmq.remoting.exception.RemotingSendRequestException; +import com.alibaba.rocketmq.remoting.exception.RemotingTimeoutException; +import com.alibaba.rocketmq.tools.admin.DefaultMQAdminExt; +import com.alibaba.rocketmq.tools.command.SubCommand; + + +/** + * 查看集群信息 + * + * @author shijia.wxr + * @since 2013-7-25 + */ +public class ClusterListSubCommand implements SubCommand { + + @Override + public String commandName() { + return "clusterList"; + } + + + @Override + public String commandDesc() { + return "List all of clusters"; + } + + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("m", "moreStats", false, "Print more stats"); + opt.setRequired(false); + options.addOption(opt); + + return options; + } + + + private void printClusterBaseInfo(final DefaultMQAdminExt defaultMQAdminExt) + throws RemotingConnectException, RemotingTimeoutException, RemotingSendRequestException, + InterruptedException, MQBrokerException { + + ClusterInfo clusterInfoSerializeWrapper = defaultMQAdminExt.examineBrokerClusterInfo(); + + System.out.printf("%-16s %-32s %-4s %-22s %-22s %11s %11s\n",// + "#Cluster Name",// + "#Broker Name",// + "#BID",// + "#Addr",// + "#Version",// + "#InTPS",// + "#OutTPS"// + ); + + Iterator>> itCluster = + clusterInfoSerializeWrapper.getClusterAddrTable().entrySet().iterator(); + while (itCluster.hasNext()) { + Map.Entry> next = itCluster.next(); + String clusterName = next.getKey(); + TreeSet brokerNameSet = new TreeSet(); + brokerNameSet.addAll(next.getValue()); + + for (String brokerName : brokerNameSet) { + BrokerData brokerData = clusterInfoSerializeWrapper.getBrokerAddrTable().get(brokerName); + if (brokerData != null) { + + Iterator> itAddr = + brokerData.getBrokerAddrs().entrySet().iterator(); + while (itAddr.hasNext()) { + Map.Entry next1 = itAddr.next(); + double in = 0; + double out = 0; + String version = ""; + + try { + KVTable kvTable = defaultMQAdminExt.fetchBrokerRuntimeStats(next1.getValue()); + String putTps = kvTable.getTable().get("putTps"); + String getTransferedTps = kvTable.getTable().get("getTransferedTps"); + version = kvTable.getTable().get("brokerVersionDesc"); + { + String[] tpss = putTps.split(" "); + if (tpss != null && tpss.length > 0) { + in = Double.parseDouble(tpss[0]); + } + } + + { + String[] tpss = getTransferedTps.split(" "); + if (tpss != null && tpss.length > 0) { + out = Double.parseDouble(tpss[0]); + } + } + } + catch (Exception e) { + } + + System.out.printf("%-16s %-32s %-4s %-22s %-22s %11.2f %11.2f\n",// + clusterName,// + brokerName,// + next1.getKey().longValue(),// + next1.getValue(),// + version,// + in,// + out// + ); + } + } + } + + if (itCluster.hasNext()) { + System.out.println(""); + } + } + } + + + private void printClusterMoreStats(final DefaultMQAdminExt defaultMQAdminExt) + throws RemotingConnectException, RemotingTimeoutException, RemotingSendRequestException, + InterruptedException, MQBrokerException { + + ClusterInfo clusterInfoSerializeWrapper = defaultMQAdminExt.examineBrokerClusterInfo(); + + System.out.printf("%-16s %-32s %14s %14s %14s %14s\n",// + "#Cluster Name",// + "#Broker Name",// + "#InTotalYest",// + "#OutTotalYest",// + "#InTotalToday",// + "#OutTotalToday"// + ); + + Iterator>> itCluster = + clusterInfoSerializeWrapper.getClusterAddrTable().entrySet().iterator(); + while (itCluster.hasNext()) { + Map.Entry> next = itCluster.next(); + String clusterName = next.getKey(); + TreeSet brokerNameSet = new TreeSet(); + brokerNameSet.addAll(next.getValue()); + + for (String brokerName : brokerNameSet) { + BrokerData brokerData = clusterInfoSerializeWrapper.getBrokerAddrTable().get(brokerName); + if (brokerData != null) { + + Iterator> itAddr = + brokerData.getBrokerAddrs().entrySet().iterator(); + while (itAddr.hasNext()) { + Map.Entry next1 = itAddr.next(); + long InTotalYest = 0; + long OutTotalYest = 0; + long InTotalToday = 0; + long OutTotalToday = 0; + + try { + KVTable kvTable = defaultMQAdminExt.fetchBrokerRuntimeStats(next1.getValue()); + String msgPutTotalYesterdayMorning = + kvTable.getTable().get("msgPutTotalYesterdayMorning"); + String msgPutTotalTodayMorning = + kvTable.getTable().get("msgPutTotalTodayMorning"); + String msgPutTotalTodayNow = kvTable.getTable().get("msgPutTotalTodayNow"); + String msgGetTotalYesterdayMorning = + kvTable.getTable().get("msgGetTotalYesterdayMorning"); + String msgGetTotalTodayMorning = + kvTable.getTable().get("msgGetTotalTodayMorning"); + String msgGetTotalTodayNow = kvTable.getTable().get("msgGetTotalTodayNow"); + + InTotalYest = + Long.parseLong(msgPutTotalTodayMorning) + - Long.parseLong(msgPutTotalYesterdayMorning); + OutTotalYest = + Long.parseLong(msgGetTotalTodayMorning) + - Long.parseLong(msgGetTotalYesterdayMorning); + + InTotalToday = + Long.parseLong(msgPutTotalTodayNow) + - Long.parseLong(msgPutTotalTodayMorning); + OutTotalToday = + Long.parseLong(msgGetTotalTodayNow) + - Long.parseLong(msgGetTotalTodayMorning); + + } + catch (Exception e) { + } + + System.out.printf("%-16s %-32s %14d %14d %14d %14d\n",// + clusterName,// + brokerName,// + InTotalYest,// + OutTotalYest,// + InTotalToday,// + OutTotalToday// + ); + } + } + } + + if (itCluster.hasNext()) { + System.out.println(""); + } + } + } + + + @Override + public void execute(final CommandLine commandLine, final Options options, RPCHook rpcHook) { + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + + try { + defaultMQAdminExt.start(); + + if (commandLine.hasOption('m')) { + this.printClusterMoreStats(defaultMQAdminExt); + } + else { + this.printClusterBaseInfo(defaultMQAdminExt); + } + } + catch (Exception e) { + e.printStackTrace(); + } + finally { + defaultMQAdminExt.shutdown(); + } + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/connection/ConsumerConnectionSubCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/connection/ConsumerConnectionSubCommand.java new file mode 100644 index 000000000..a997935dc --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/connection/ConsumerConnectionSubCommand.java @@ -0,0 +1,116 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.tools.command.connection; + +import java.util.Iterator; +import java.util.Map.Entry; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; + +import com.alibaba.rocketmq.common.MQVersion; +import com.alibaba.rocketmq.common.protocol.body.Connection; +import com.alibaba.rocketmq.common.protocol.body.ConsumerConnection; +import com.alibaba.rocketmq.common.protocol.heartbeat.SubscriptionData; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.tools.admin.DefaultMQAdminExt; +import com.alibaba.rocketmq.tools.command.SubCommand; + + +/** + * 查询Consumer的网络连接，以及客户端版本号，订阅关系等 + * + * @author shijia.wxr + * @since 2013-10-13 + */ +public class ConsumerConnectionSubCommand implements SubCommand { + + @Override + public String commandName() { + return "consumerConnection"; + } + + + @Override + public String commandDesc() { + return "Query consumer's socket connection, client version and subscription"; + } + + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("g", "consumerGroup", true, "consumer group name"); + opt.setRequired(true); + options.addOption(opt); + + return options; + } + + + @Override + public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) { + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + + try { + defaultMQAdminExt.start(); + + String group = commandLine.getOptionValue('g').trim(); + + ConsumerConnection cc = defaultMQAdminExt.examineConsumerConnectionInfo(group); + + // 打印连接 + int i = 1; + for (Connection conn : cc.getConnectionSet()) { + System.out.printf("%03d %-32s %-22s %-8s %s\n",// + i++,// + conn.getClientId(),// + conn.getClientAddr(),// + conn.getLanguage(),// + MQVersion.getVersionDesc(conn.getVersion())// + ); + } + + // 打印订阅关系 + System.out.println("\nBelow is subscription:"); + Iterator> it = cc.getSubscriptionTable().entrySet().iterator(); + i = 1; + while (it.hasNext()) { + Entry entry = it.next(); + SubscriptionData sd = entry.getValue(); + System.out.printf("%03d Topic: %-40s SubExpression: %s\n",// + i++,// + sd.getTopic(),// + sd.getSubString()// + ); + } + + // 打印其他订阅参数 + System.out.println(""); + System.out.printf("ConsumeType: %s\n", cc.getConsumeType()); + System.out.printf("MessageModel: %s\n", cc.getMessageModel()); + System.out.printf("ConsumeFromWhere: %s\n", cc.getConsumeFromWhere()); + } + catch (Exception e) { + e.printStackTrace(); + } + finally { + defaultMQAdminExt.shutdown(); + } + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/connection/ProducerConnectionSubCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/connection/ProducerConnectionSubCommand.java new file mode 100644 index 000000000..9283bada4 --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/connection/ProducerConnectionSubCommand.java @@ -0,0 +1,97 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.tools.command.connection; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; + +import com.alibaba.rocketmq.common.MQVersion; +import com.alibaba.rocketmq.common.protocol.body.Connection; +import com.alibaba.rocketmq.common.protocol.body.ProducerConnection; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.tools.admin.DefaultMQAdminExt; +import com.alibaba.rocketmq.tools.command.SubCommand; + + +/** + * 查询Producer的网络连接 + * + * @author shijia.wxr + * @since 2013-10-13 + */ +public class ProducerConnectionSubCommand implements SubCommand { + + @Override + public String commandName() { + return "producerConnection"; + } + + + @Override + public String commandDesc() { + return "Query producer's socket connection and client version"; + } + + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("g", "producerGroup", true, "producer group name"); + opt.setRequired(true); + options.addOption(opt); + + // topic必须设置 + opt = new Option("t", "topic", true, "topic name"); + opt.setRequired(true); + options.addOption(opt); + + return options; + } + + + @Override + public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) { + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + + try { + defaultMQAdminExt.start(); + + String group = commandLine.getOptionValue('g').trim(); + String topic = commandLine.getOptionValue('t').trim(); + + ProducerConnection pc = defaultMQAdminExt.examineProducerConnectionInfo(group, topic); + + int i = 1; + for (Connection conn : pc.getConnectionSet()) { + System.out.printf("%04d %-32s %-22s %-8s %s\n",// + i++,// + conn.getClientId(),// + conn.getClientAddr(),// + conn.getLanguage(),// + MQVersion.getVersionDesc(conn.getVersion())// + ); + } + } + catch (Exception e) { + e.printStackTrace(); + } + finally { + defaultMQAdminExt.shutdown(); + } + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/consumer/ConsumerProgressSubCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/consumer/ConsumerProgressSubCommand.java new file mode 100644 index 000000000..04e43f1cc --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/consumer/ConsumerProgressSubCommand.java @@ -0,0 +1,321 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.tools.command.consumer; + +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.slf4j.Logger; + +import com.alibaba.rocketmq.client.log.ClientLogger; +import com.alibaba.rocketmq.common.MQVersion; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.admin.ConsumeStats; +import com.alibaba.rocketmq.common.admin.OffsetWrapper; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.common.protocol.body.ConsumerConnection; +import com.alibaba.rocketmq.common.protocol.body.TopicList; +import com.alibaba.rocketmq.common.protocol.heartbeat.ConsumeType; +import com.alibaba.rocketmq.common.protocol.heartbeat.MessageModel; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.tools.admin.DefaultMQAdminExt; +import com.alibaba.rocketmq.tools.command.SubCommand; + + +/** + * 查看订阅组消费状态，消费进度 + * + * @author shijia.wxr + * @since 2013-8-11 + */ +public class ConsumerProgressSubCommand implements SubCommand { + private final Logger log = ClientLogger.getLog(); + + + @Override + public String commandName() { + return "consumerProgress"; + } + + + @Override + public String commandDesc() { + return "Query consumers's progress, speed"; + } + + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("g", "groupName", true, "consumer group name"); + opt.setRequired(false); + options.addOption(opt); + + return options; + } + + + @Override + public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) { + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + + try { + defaultMQAdminExt.start(); + + // 查询特定consumer + if (commandLine.hasOption('g')) { + String consumerGroup = commandLine.getOptionValue('g').trim(); + ConsumeStats consumeStats = defaultMQAdminExt.examineConsumeStats(consumerGroup); + + List mqList = new LinkedList(); + mqList.addAll(consumeStats.getOffsetTable().keySet()); + Collections.sort(mqList); + + System.out.printf("%-32s %-32s %-4s %-20s %-20s %s\n",// + "#Topic",// + "#Broker Name",// + "#QID",// + "#Broker Offset",// + "#Consumer Offset",// + "#Diff" // + ); + + long diffTotal = 0L; + + for (MessageQueue mq : mqList) { + OffsetWrapper offsetWrapper = consumeStats.getOffsetTable().get(mq); + + long diff = offsetWrapper.getBrokerOffset() - offsetWrapper.getConsumerOffset(); + diffTotal += diff; + + System.out.printf("%-32s %-32s %-4d %-20d %-20d %d\n",// + UtilAll.frontStringAtLeast(mq.getTopic(), 32),// + UtilAll.frontStringAtLeast(mq.getBrokerName(), 32),// + mq.getQueueId(),// + offsetWrapper.getBrokerOffset(),// + offsetWrapper.getConsumerOffset(),// + diff // + ); + } + + System.out.println(""); + System.out.printf("Consume TPS: %d\n", consumeStats.getConsumeTps()); + System.out.printf("Diff Total: %d\n", diffTotal); + } + // 查询全部 + else { + System.out.printf("%-32s %-6s %-24s %-5s %-14s %-7s %s\n",// + "#Group",// + "#Count",// + "#Version",// + "#Type",// + "#Model",// + "#TPS",// + "#Diff Total"// + ); + + List groupConsumeInfoList = new LinkedList(); + + TopicList topicList = defaultMQAdminExt.fetchAllTopicList(); + for (String topic : topicList.getTopicList()) { + if (topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) { + String consumerGroup = topic.substring(MixAll.RETRY_GROUP_TOPIC_PREFIX.length()); + + try { + ConsumeStats consumeStats = null; + try { + consumeStats = defaultMQAdminExt.examineConsumeStats(consumerGroup); + } + catch (Exception e) { + log.warn("examineConsumeStats exception, " + consumerGroup, e); + } + + ConsumerConnection cc = null; + try { + cc = defaultMQAdminExt.examineConsumerConnectionInfo(consumerGroup); + } + catch (Exception e) { + log.warn("examineConsumerConnectionInfo exception, " + consumerGroup, e); + } + + GroupConsumeInfo groupConsumeInfo = new GroupConsumeInfo(); + groupConsumeInfo.setGroup(consumerGroup); + + if (consumeStats != null) { + groupConsumeInfo.setConsumeTps((int) consumeStats.getConsumeTps()); + groupConsumeInfo.setDiffTotal(consumeStats.computeTotalDiff()); + } + + if (cc != null) { + groupConsumeInfo.setCount(cc.getConnectionSet().size()); + groupConsumeInfo.setMessageModel(cc.getMessageModel()); + groupConsumeInfo.setConsumeType(cc.getConsumeType()); + groupConsumeInfo.setVersion(cc.computeMinVersion()); + } + + groupConsumeInfoList.add(groupConsumeInfo); + } + catch (Exception e) { + log.warn("examineConsumeStats or examineConsumerConnectionInfo exception, " + + consumerGroup, e); + } + } + } + + Collections.sort(groupConsumeInfoList); + + for (GroupConsumeInfo info : groupConsumeInfoList) { + System.out.printf("%-32s %-6d %-24s %-5s %-14s %-7d %d\n",// + UtilAll.frontStringAtLeast(info.getGroup(), 32),// + info.getCount(),// + info.versionDesc(),// + info.consumeTypeDesc(),// + info.messageModelDesc(),// + info.getConsumeTps(),// + info.getDiffTotal()// + ); + } + } + } + catch (Exception e) { + e.printStackTrace(); + } + finally { + defaultMQAdminExt.shutdown(); + } + } +} + + +class GroupConsumeInfo implements Comparable { + private String group; + private int version; + private int count; + private ConsumeType consumeType; + private MessageModel messageModel; + private int consumeTps; + private long diffTotal; + + + public String getGroup() { + return group; + } + + + public String consumeTypeDesc() { + if (this.count != 0) { + return this.getConsumeType() == ConsumeType.CONSUME_ACTIVELY ? "PULL" : "PUSH"; + } + return ""; + } + + + public String messageModelDesc() { + if (this.count != 0 && this.getConsumeType() == ConsumeType.CONSUME_PASSIVELY) { + return this.getMessageModel().toString(); + } + return ""; + } + + + public String versionDesc() { + if (this.count != 0) { + return MQVersion.getVersionDesc(this.version); + } + return ""; + } + + + public void setGroup(String group) { + this.group = group; + } + + + public int getCount() { + return count; + } + + + public void setCount(int count) { + this.count = count; + } + + + public ConsumeType getConsumeType() { + return consumeType; + } + + + public void setConsumeType(ConsumeType consumeType) { + this.consumeType = consumeType; + } + + + public MessageModel getMessageModel() { + return messageModel; + } + + + public void setMessageModel(MessageModel messageModel) { + this.messageModel = messageModel; + } + + + public long getDiffTotal() { + return diffTotal; + } + + + public void setDiffTotal(long diffTotal) { + this.diffTotal = diffTotal; + } + + + @Override + public int compareTo(GroupConsumeInfo o) { + if (this.count != o.count) { + return o.count - this.count; + } + + return (int) (o.diffTotal - diffTotal); + } + + + public int getConsumeTps() { + return consumeTps; + } + + + public void setConsumeTps(int consumeTps) { + this.consumeTps = consumeTps; + } + + + public int getVersion() { + return version; + } + + + public void setVersion(int version) { + this.version = version; + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/consumer/ConsumerStatusSubCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/consumer/ConsumerStatusSubCommand.java new file mode 100644 index 000000000..2c6ca7032 --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/consumer/ConsumerStatusSubCommand.java @@ -0,0 +1,166 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.tools.command.consumer; + +import java.util.Iterator; +import java.util.Map.Entry; +import java.util.TreeMap; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; + +import com.alibaba.rocketmq.common.MQVersion; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.protocol.body.Connection; +import com.alibaba.rocketmq.common.protocol.body.ConsumerConnection; +import com.alibaba.rocketmq.common.protocol.body.ConsumerRunningInfo; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.tools.admin.DefaultMQAdminExt; +import com.alibaba.rocketmq.tools.command.MQAdminStartup; +import com.alibaba.rocketmq.tools.command.SubCommand; + + +/** + * 查询Consumer内部数据结构 + * + * @author shijia.wxr + * @since 2014-07-20 + */ +public class ConsumerStatusSubCommand implements SubCommand { + + @Override + public String commandName() { + return "consumerStatus"; + } + + + @Override + public String commandDesc() { + return "Query consumer's internal data structure"; + } + + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("g", "consumerGroup", true, "consumer group name"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("i", "clientId", true, "The consumer's client id"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("s", "jstack", false, "Run jstack command in the consumer progress"); + opt.setRequired(false); + options.addOption(opt); + + return options; + } + + + @Override + public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) { + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + + try { + defaultMQAdminExt.start(); + + String group = commandLine.getOptionValue('g').trim(); + + ConsumerConnection cc = defaultMQAdminExt.examineConsumerConnectionInfo(group); + + boolean jstack = commandLine.hasOption('s'); + + if (!commandLine.hasOption('i')) { + // 打印连接 + int i = 1; + long now = System.currentTimeMillis(); + final TreeMap criTable = + new TreeMap(); + for (Connection conn : cc.getConnectionSet()) { + try { + ConsumerRunningInfo consumerRunningInfo = + defaultMQAdminExt.getConsumerRunningInfo(group, conn.getClientId(), jstack); + if (consumerRunningInfo != null) { + criTable.put(conn.getClientId(), consumerRunningInfo); + String filePath = now + "/" + conn.getClientId(); + MixAll.string2FileNotSafe(consumerRunningInfo.formatString(), filePath); + System.out.printf("%03d %-40s %-20s %s\n",// + i++,// + conn.getClientId(),// + MQVersion.getVersionDesc(conn.getVersion()),// + filePath); + } + } + catch (Exception e) { + e.printStackTrace(); + } + } + + if (!criTable.isEmpty()) { + boolean subSame = ConsumerRunningInfo.analyzeSubscription(criTable); + + boolean rebalanceOK = subSame && ConsumerRunningInfo.analyzeRebalance(criTable); + + if (subSame) { + System.out.println("\n\nSame subscription in the same group of consumer"); + + System.out.printf("\n\nRebalance %s\n", rebalanceOK ? "OK" : "Failed"); + + Iterator> it = criTable.entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + String result = + ConsumerRunningInfo.analyzeProcessQueue(next.getKey(), next.getValue()); + if (result.length() > 0) { + System.out.println(result); + } + } + } + else { + System.out + .println("\n\nWARN: Different subscription in the same group of consumer!!!"); + } + } + } + else { + String clientId = commandLine.getOptionValue('i').trim(); + ConsumerRunningInfo consumerRunningInfo = + defaultMQAdminExt.getConsumerRunningInfo(group, clientId, jstack); + if (consumerRunningInfo != null) { + System.out.println(consumerRunningInfo.formatString()); + } + } + } + catch (Exception e) { + e.printStackTrace(); + } + finally { + defaultMQAdminExt.shutdown(); + } + } + + + public static void main(String[] args) { + System.setProperty(MixAll.NAMESRV_ADDR_PROPERTY, "127.0.0.1:9876"); + MQAdminStartup.main(new String[] { new ConsumerStatusSubCommand().commandName(), // + "-g", "benchmark_consumer" // + }); + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/consumer/DeleteSubscriptionGroupCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/consumer/DeleteSubscriptionGroupCommand.java new file mode 100644 index 000000000..e65fa30c1 --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/consumer/DeleteSubscriptionGroupCommand.java @@ -0,0 +1,121 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.tools.command.consumer; + +import java.util.Set; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; + +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.srvutil.ServerUtil; +import com.alibaba.rocketmq.tools.admin.DefaultMQAdminExt; +import com.alibaba.rocketmq.tools.command.CommandUtil; +import com.alibaba.rocketmq.tools.command.SubCommand; +import com.alibaba.rocketmq.tools.command.topic.DeleteTopicSubCommand; + + +/** + * 删除订阅组配置命令 + * + * @author manhong.yqd + * @since 2013-8-22 + */ +public class DeleteSubscriptionGroupCommand implements SubCommand { + @Override + public String commandName() { + return "deleteSubGroup"; + } + + + @Override + public String commandDesc() { + return "Delete subscription group from broker."; + } + + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("b", "brokerAddr", true, "delete subscription group from which broker"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("c", "clusterName", true, "delete subscription group from which cluster"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("g", "groupName", true, "subscription group name"); + opt.setRequired(true); + options.addOption(opt); + + return options; + } + + + @Override + public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) { + DefaultMQAdminExt adminExt = new DefaultMQAdminExt(rpcHook); + adminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + try { + // groupName + String groupName = commandLine.getOptionValue('g').trim(); + + if (commandLine.hasOption('b')) { + String addr = commandLine.getOptionValue('b').trim(); + adminExt.start(); + + adminExt.deleteSubscriptionGroup(addr, groupName); + System.out.printf("delete subscription group [%s] from broker [%s] success.\n", groupName, + addr); + + return; + } + else if (commandLine.hasOption('c')) { + String clusterName = commandLine.getOptionValue('c').trim(); + adminExt.start(); + + Set masterSet = CommandUtil.fetchMasterAddrByClusterName(adminExt, clusterName); + for (String master : masterSet) { + adminExt.deleteSubscriptionGroup(master, groupName); + System.out.printf( + "delete subscription group [%s] from broker [%s] in cluster [%s] success.\n", + groupName, master, clusterName); + } + + // 删除%RETRY%打头的Topic + try { + DeleteTopicSubCommand.deleteTopic(adminExt, clusterName, MixAll.RETRY_GROUP_TOPIC_PREFIX + + groupName); + DeleteTopicSubCommand.deleteTopic(adminExt, clusterName, MixAll.DLQ_GROUP_TOPIC_PREFIX + + groupName); + } + catch (Exception e) { + } + return; + } + + ServerUtil.printCommandLineHelp("mqadmin " + this.commandName(), options); + } + catch (Exception e) { + e.printStackTrace(); + } + finally { + adminExt.shutdown(); + } + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/consumer/StartMonitoringSubCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/consumer/StartMonitoringSubCommand.java new file mode 100644 index 000000000..14ce9f640 --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/consumer/StartMonitoringSubCommand.java @@ -0,0 +1,72 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.tools.command.consumer; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Options; +import org.slf4j.Logger; + +import com.alibaba.rocketmq.client.log.ClientLogger; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.tools.command.SubCommand; +import com.alibaba.rocketmq.tools.monitor.DefaultMonitorListener; +import com.alibaba.rocketmq.tools.monitor.MonitorConfig; +import com.alibaba.rocketmq.tools.monitor.MonitorService; + + +/** + * 启动监控 + * + * @author shijia.wxr + * @since 2014-7-5 + */ +public class StartMonitoringSubCommand implements SubCommand { + private final Logger log = ClientLogger.getLog(); + + + @Override + public String commandName() { + return "startMonitoring"; + } + + + @Override + public String commandDesc() { + return "Start Monitoring"; + } + + + @Override + public Options buildCommandlineOptions(Options options) { + return options; + } + + + @Override + public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) { + try { + MonitorService monitorService = + new MonitorService(new MonitorConfig(), new DefaultMonitorListener(), rpcHook); + + monitorService.start(); + } + catch (Exception e) { + e.printStackTrace(); + } + finally { + } + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/consumer/UpdateSubGroupSubCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/consumer/UpdateSubGroupSubCommand.java new file mode 100644 index 000000000..c64e6a530 --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/consumer/UpdateSubGroupSubCommand.java @@ -0,0 +1,188 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.tools.command.consumer; + +import java.util.Set; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; + +import com.alibaba.rocketmq.common.subscription.SubscriptionGroupConfig; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.srvutil.ServerUtil; +import com.alibaba.rocketmq.tools.admin.DefaultMQAdminExt; +import com.alibaba.rocketmq.tools.command.CommandUtil; +import com.alibaba.rocketmq.tools.command.SubCommand; + + +/** + * 修改、创建订阅组配置命令 + * + * @author shijia.wxr + * @since 2013-7-21 + */ +public class UpdateSubGroupSubCommand implements SubCommand { + + @Override + public String commandName() { + return "updateSubGroup"; + } + + + @Override + public String commandDesc() { + return "Update or create subscription group"; + } + + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("b", "brokerAddr", true, "create subscription group to which broker"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("c", "clusterName", true, "create subscription group to which cluster"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("g", "groupName", true, "consumer group name"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("s", "consumeEnable", true, "consume enable"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("m", "consumeFromMinEnable", true, "from min offset"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("d", "consumeBroadcastEnable", true, "broadcast"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("q", "retryQueueNums", true, "retry queue nums"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("r", "retryMaxTimes", true, "retry max times"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("i", "brokerId", true, "consumer from which broker id"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("w", "whichBrokerWhenConsumeSlowly", true, "which broker id when consume slowly"); + opt.setRequired(false); + options.addOption(opt); + + return options; + } + + + @Override + public void execute(final CommandLine commandLine, final Options options, RPCHook rpcHook) { + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + + try { + SubscriptionGroupConfig subscriptionGroupConfig = new SubscriptionGroupConfig(); + subscriptionGroupConfig.setConsumeBroadcastEnable(false); + subscriptionGroupConfig.setConsumeFromMinEnable(false); + + // groupName + subscriptionGroupConfig.setGroupName(commandLine.getOptionValue('g').trim()); + + // consumeEnable + if (commandLine.hasOption('s')) { + subscriptionGroupConfig.setConsumeEnable(Boolean.parseBoolean(commandLine.getOptionValue('s') + .trim())); + } + + // consumeFromMinEnable + if (commandLine.hasOption('m')) { + subscriptionGroupConfig.setConsumeFromMinEnable(Boolean.parseBoolean(commandLine + .getOptionValue('m').trim())); + } + + // consumeBroadcastEnable + if (commandLine.hasOption('d')) { + subscriptionGroupConfig.setConsumeBroadcastEnable(Boolean.parseBoolean(commandLine + .getOptionValue('d').trim())); + } + + // retryQueueNums + if (commandLine.hasOption('q')) { + subscriptionGroupConfig.setRetryQueueNums(Integer.parseInt(commandLine.getOptionValue('q') + .trim())); + } + + // retryMaxTimes + if (commandLine.hasOption('r')) { + subscriptionGroupConfig.setRetryMaxTimes(Integer.parseInt(commandLine.getOptionValue('r') + .trim())); + } + + // brokerId + if (commandLine.hasOption('i')) { + subscriptionGroupConfig.setBrokerId(Long.parseLong(commandLine.getOptionValue('i').trim())); + } + + // whichBrokerWhenConsumeSlowly + if (commandLine.hasOption('w')) { + subscriptionGroupConfig.setWhichBrokerWhenConsumeSlowly(Long.parseLong(commandLine + .getOptionValue('w').trim())); + } + + if (commandLine.hasOption('b')) { + String addr = commandLine.getOptionValue('b').trim(); + + defaultMQAdminExt.start(); + + defaultMQAdminExt.createAndUpdateSubscriptionGroupConfig(addr, subscriptionGroupConfig); + System.out.printf("create subscription group to %s success.\n", addr); + System.out.println(subscriptionGroupConfig); + return; + + } + else if (commandLine.hasOption('c')) { + String clusterName = commandLine.getOptionValue('c').trim(); + + defaultMQAdminExt.start(); + + Set masterSet = + CommandUtil.fetchMasterAddrByClusterName(defaultMQAdminExt, clusterName); + for (String addr : masterSet) { + defaultMQAdminExt.createAndUpdateSubscriptionGroupConfig(addr, subscriptionGroupConfig); + System.out.printf("create subscription group to %s success.\n", addr); + } + System.out.println(subscriptionGroupConfig); + return; + } + + ServerUtil.printCommandLineHelp("mqadmin " + this.commandName(), options); + } + catch (Exception e) { + e.printStackTrace(); + } + finally { + defaultMQAdminExt.shutdown(); + } + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/message/CheckMsgSubCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/message/CheckMsgSubCommand.java new file mode 100644 index 000000000..3a336757d --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/message/CheckMsgSubCommand.java @@ -0,0 +1,58 @@ +package com.alibaba.rocketmq.tools.command.message; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; + +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.tools.command.SubCommand; + + +/** + * @auther lansheng.zj + */ +public class CheckMsgSubCommand implements SubCommand { + @Override + public String commandName() { + return "checkMsg"; + } + + + @Override + public String commandDesc() { + return "Check Message Store"; + } + + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("p", "cStorePath", true, "cStorePath"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("s", "cSize ", true, "cSize"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("l", "lStorePath ", true, "lStorePath"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("z", "lSize ", true, "lSize"); + opt.setRequired(true); + options.addOption(opt); + + return options; + } + + + @Override + public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) { + Store store = new Store(commandLine.getOptionValue("cStorePath").trim(), // + Integer.parseInt(commandLine.getOptionValue("cSize").trim()),// + commandLine.getOptionValue("lStorePath").trim(), // + Integer.parseInt(commandLine.getOptionValue("lSize").trim())); + store.load(); + store.traval(false); + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/message/PrintMessageSubCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/message/PrintMessageSubCommand.java new file mode 100644 index 000000000..525e3e2dc --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/message/PrintMessageSubCommand.java @@ -0,0 +1,173 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.tools.command.message; + +import java.io.UnsupportedEncodingException; +import java.util.List; +import java.util.Set; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; + +import com.alibaba.rocketmq.client.consumer.DefaultMQPullConsumer; +import com.alibaba.rocketmq.client.consumer.PullResult; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.message.MessageExt; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.tools.command.SubCommand; + + +/** + * 打印指定Topic的所有消息，某个时间区间，方便排查问题 + * + * @author shijia.wxr + * @since 2014-6-22 + */ +public class PrintMessageSubCommand implements SubCommand { + + @Override + public String commandName() { + return "printMsg"; + } + + + @Override + public String commandDesc() { + return "Print Message Detail"; + } + + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("t", "topic", true, "topic name"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("c", "charsetName ", true, "CharsetName(eg: UTF-8、GBK)"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("s", "subExpression ", true, "Subscribe Expression(eg: TagA || TagB)"); + opt.setRequired(false); + options.addOption(opt); + + opt = + new Option("b", "beginTimestamp ", true, + "Begin timestamp[currentTimeMillis|yyyy-MM-dd#HH:mm:ss:SSS]"); + opt.setRequired(false); + options.addOption(opt); + + opt = + new Option("e", "endTimestamp ", true, + "End timestamp[currentTimeMillis|yyyy-MM-dd#HH:mm:ss:SSS]"); + opt.setRequired(false); + options.addOption(opt); + + return options; + } + + + public static void printMessage(final List msgs, final String charsetName) { + for (MessageExt msg : msgs) { + try { + System.out.printf("MSGID: %s %s BODY: %s\n", msg.getMsgId(), msg.toString(), + new String(msg.getBody(), charsetName)); + } + catch (UnsupportedEncodingException e) { + } + } + } + + + public static long timestampFormat(final String value) { + long timestamp = 0; + try { + // 直接输入 long 类型的 timestamp + timestamp = Long.valueOf(value); + } + catch (NumberFormatException e) { + // 输入的为日期格式，精确到毫秒 + timestamp = UtilAll.parseDate(value, UtilAll.yyyy_MM_dd_HH_mm_ss_SSS).getTime(); + } + + return timestamp; + } + + + @Override + public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) { + DefaultMQPullConsumer consumer = new DefaultMQPullConsumer(MixAll.TOOLS_CONSUMER_GROUP, rpcHook); + + try { + String topic = commandLine.getOptionValue('t').trim(); + + String charsetName = // + !commandLine.hasOption('c') ? "UTF-8" : commandLine.getOptionValue('c').trim(); + + String subExpression = // + !commandLine.hasOption('s') ? "*" : commandLine.getOptionValue('s').trim(); + + consumer.start(); + + Set mqs = consumer.fetchSubscribeMessageQueues(topic); + for (MessageQueue mq : mqs) { + long minOffset = consumer.minOffset(mq); + long maxOffset = consumer.maxOffset(mq); + + if (commandLine.hasOption('b')) { + String timestampStr = commandLine.getOptionValue('b').trim(); + long timeValue = timestampFormat(timestampStr); + minOffset = consumer.searchOffset(mq, timeValue); + } + + if (commandLine.hasOption('e')) { + String timestampStr = commandLine.getOptionValue('e').trim(); + long timeValue = timestampFormat(timestampStr); + maxOffset = consumer.searchOffset(mq, timeValue); + } + + READQ: for (long offset = minOffset; offset < maxOffset;) { + try { + PullResult pullResult = consumer.pull(mq, subExpression, offset, 32); + offset = pullResult.getNextBeginOffset(); + switch (pullResult.getPullStatus()) { + case FOUND: + printMessage(pullResult.getMsgFoundList(), charsetName); + break; + case NO_MATCHED_MSG: + case NO_NEW_MSG: + case OFFSET_ILLEGAL: + break READQ; + } + } + catch (Exception e) { + break; + } + } + } + + } + catch (Exception e) { + e.printStackTrace(); + } + finally { + consumer.shutdown(); + } + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/message/QueryMsgByIdSubCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/message/QueryMsgByIdSubCommand.java new file mode 100644 index 000000000..2487b03b4 --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/message/QueryMsgByIdSubCommand.java @@ -0,0 +1,236 @@ +/** + + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.tools.command.message; + +import java.io.DataOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.List; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; + +import com.alibaba.rocketmq.client.exception.MQBrokerException; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.message.MessageExt; +import com.alibaba.rocketmq.common.protocol.body.ConsumeMessageDirectlyResult; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.remoting.common.RemotingHelper; +import com.alibaba.rocketmq.remoting.exception.RemotingException; +import com.alibaba.rocketmq.tools.admin.DefaultMQAdminExt; +import com.alibaba.rocketmq.tools.admin.api.MessageTrack; +import com.alibaba.rocketmq.tools.command.MQAdminStartup; +import com.alibaba.rocketmq.tools.command.SubCommand; + + +/** + * 根据消息Id查询消息 + * + * @author shijia.wxr + * @since 2013-8-12 + */ +public class QueryMsgByIdSubCommand implements SubCommand { + + @Override + public String commandName() { + return "queryMsgById"; + } + + + @Override + public String commandDesc() { + return "Query Message by Id"; + } + + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("i", "msgId", true, "Message Id"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("g", "consumerGroup", true, "consumer group name"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("d", "clientId", true, "The consumer's client id"); + opt.setRequired(false); + options.addOption(opt); + + return options; + } + + + public static void queryById(final DefaultMQAdminExt admin, final String msgId) throws MQClientException, + RemotingException, MQBrokerException, InterruptedException, IOException { + MessageExt msg = admin.viewMessage(msgId); + + // 存储消息 body 到指定路径 + String bodyTmpFilePath = createBodyFile(msg); + + System.out.printf("%-20s %s\n",// + "Topic:",// + msg.getTopic()// + ); + + System.out.printf("%-20s %s\n",// + "Tags:",// + "[" + msg.getTags() + "]"// + ); + + System.out.printf("%-20s %s\n",// + "Keys:",// + "[" + msg.getKeys() + "]"// + ); + + System.out.printf("%-20s %d\n",// + "Queue ID:",// + msg.getQueueId()// + ); + + System.out.printf("%-20s %d\n",// + "Queue Offset:",// + msg.getQueueOffset()// + ); + + System.out.printf("%-20s %d\n",// + "CommitLog Offset:",// + msg.getCommitLogOffset()// + ); + + System.out.printf("%-20s %d\n",// + "Reconsume Times:",// + msg.getReconsumeTimes()// + ); + + System.out.printf("%-20s %s\n",// + "Born Timestamp:",// + UtilAll.timeMillisToHumanString2(msg.getBornTimestamp())// + ); + + System.out.printf("%-20s %s\n",// + "Store Timestamp:",// + UtilAll.timeMillisToHumanString2(msg.getStoreTimestamp())// + ); + + System.out.printf("%-20s %s\n",// + "Born Host:",// + RemotingHelper.parseSocketAddressAddr(msg.getBornHost())// + ); + + System.out.printf("%-20s %s\n",// + "Store Host:",// + RemotingHelper.parseSocketAddressAddr(msg.getStoreHost())// + ); + + System.out.printf("%-20s %d\n",// + "System Flag:",// + msg.getSysFlag()// + ); + + System.out.printf("%-20s %s\n",// + "Properties:",// + msg.getProperties() != null ? msg.getProperties().toString() : ""// + ); + + System.out.printf("%-20s %s\n",// + "Message Body Path:",// + bodyTmpFilePath// + ); + + try { + List mtdList = admin.messageTrackDetail(msg); + if (mtdList.isEmpty()) { + System.out.println("\n\nWARN: No Consumer"); + } + else { + System.out.println("\n\n"); + for (MessageTrack mt : mtdList) { + System.out.println(mt); + } + } + } + catch (Exception e) { + e.printStackTrace(); + } + } + + + @Override + public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) { + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + + try { + defaultMQAdminExt.start(); + + final String msgId = commandLine.getOptionValue('i').trim(); + if (commandLine.hasOption('g') && commandLine.hasOption('d')) { + final String consumerGroup = commandLine.getOptionValue('g').trim(); + final String clientId = commandLine.getOptionValue('d').trim(); + ConsumeMessageDirectlyResult result = + defaultMQAdminExt.consumeMessageDirectly(consumerGroup, clientId, msgId); + System.out.println(result); + } + else { + + queryById(defaultMQAdminExt, msgId); + } + } + catch (Exception e) { + e.printStackTrace(); + } + finally { + defaultMQAdminExt.shutdown(); + } + } + + + private static String createBodyFile(MessageExt msg) throws IOException { + DataOutputStream dos = null; + + try { + String bodyTmpFilePath = "/tmp/rocketmq/msgbodys"; + File file = new File(bodyTmpFilePath); + if (!file.exists()) { + file.mkdirs(); + } + bodyTmpFilePath = bodyTmpFilePath + "/" + msg.getMsgId(); + dos = new DataOutputStream(new FileOutputStream(bodyTmpFilePath)); + dos.write(msg.getBody()); + return bodyTmpFilePath; + } + finally { + if (dos != null) + dos.close(); + } + } + + + public static void main(String[] args) { + MQAdminStartup.main(new String[] { new QueryMsgByIdSubCommand().commandName(), // + "-n", "127.0.0.1:9876", // + "-g", "CID_110", // + "-d", "127.0.0.1@73376", // + "-i", "0A654A3400002ABD00000011C3555205" // + }); + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/message/QueryMsgByKeySubCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/message/QueryMsgByKeySubCommand.java new file mode 100644 index 000000000..55b7d7122 --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/message/QueryMsgByKeySubCommand.java @@ -0,0 +1,98 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.tools.command.message; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; + +import com.alibaba.rocketmq.client.QueryResult; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.common.message.MessageExt; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.tools.admin.DefaultMQAdminExt; +import com.alibaba.rocketmq.tools.command.SubCommand; + + +/** + * 根据消息Key查询消息 + * + * @author shijia.wxr + * @since 2013-8-12 + */ +public class QueryMsgByKeySubCommand implements SubCommand { + + @Override + public String commandName() { + return "queryMsgByKey"; + } + + + @Override + public String commandDesc() { + return "Query Message by Key"; + } + + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("t", "topic", true, "topic name"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("k", "msgKey", true, "Message Key"); + opt.setRequired(true); + options.addOption(opt); + + return options; + } + + + void queryByKey(final DefaultMQAdminExt admin, final String topic, final String key) + throws MQClientException, InterruptedException { + admin.start(); + + QueryResult queryResult = admin.queryMessage(topic, key, 64, 0, Long.MAX_VALUE); + System.out.printf("%-50s %4s %40s\n",// + "#Message ID",// + "#QID",// + "#Offset"); + for (MessageExt msg : queryResult.getMessageList()) { + System.out.printf("%-50s %4d %40d\n", msg.getMsgId(), msg.getQueueId(), msg.getQueueOffset()); + } + } + + + @Override + public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) { + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + + try { + final String topic = commandLine.getOptionValue('t').trim(); + final String key = commandLine.getOptionValue('k').trim(); + + this.queryByKey(defaultMQAdminExt, topic, key); + } + catch (Exception e) { + e.printStackTrace(); + } + finally { + defaultMQAdminExt.shutdown(); + } + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/message/QueryMsgByOffsetSubCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/message/QueryMsgByOffsetSubCommand.java new file mode 100644 index 000000000..96ee1073e --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/message/QueryMsgByOffsetSubCommand.java @@ -0,0 +1,118 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.tools.command.message; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; + +import com.alibaba.rocketmq.client.consumer.DefaultMQPullConsumer; +import com.alibaba.rocketmq.client.consumer.PullResult; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.tools.admin.DefaultMQAdminExt; +import com.alibaba.rocketmq.tools.command.SubCommand; + + +/** + * 根据消息Offset查询消息 + * + * @author shijia.wxr + * @since 2013-8-12 + */ +public class QueryMsgByOffsetSubCommand implements SubCommand { + + @Override + public String commandName() { + return "queryMsgByOffset"; + } + + + @Override + public String commandDesc() { + return "Query Message by offset"; + } + + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("t", "topic", true, "topic name"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("b", "brokerName", true, "Broker Name"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("i", "queueId", true, "Queue Id"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("o", "offset", true, "Queue Offset"); + opt.setRequired(true); + options.addOption(opt); + + return options; + } + + + @Override + public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) { + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + DefaultMQPullConsumer defaultMQPullConsumer = new DefaultMQPullConsumer(MixAll.TOOLS_CONSUMER_GROUP); + + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + defaultMQPullConsumer.setInstanceName(Long.toString(System.currentTimeMillis())); + + try { + String topic = commandLine.getOptionValue('t').trim(); + String brokerName = commandLine.getOptionValue('b').trim(); + String queueId = commandLine.getOptionValue('i').trim(); + String offset = commandLine.getOptionValue('o').trim(); + + MessageQueue mq = new MessageQueue(); + mq.setTopic(topic); + mq.setBrokerName(brokerName); + mq.setQueueId(Integer.parseInt(queueId)); + + defaultMQPullConsumer.start(); + defaultMQAdminExt.start(); + + PullResult pullResult = defaultMQPullConsumer.pull(mq, "*", Long.parseLong(offset), 1); + if (pullResult != null) { + switch (pullResult.getPullStatus()) { + case FOUND: + QueryMsgByIdSubCommand.queryById(defaultMQAdminExt, pullResult.getMsgFoundList().get(0) + .getMsgId()); + break; + case NO_MATCHED_MSG: + case NO_NEW_MSG: + case OFFSET_ILLEGAL: + default: + break; + } + } + } + catch (Exception e) { + e.printStackTrace(); + } + finally { + defaultMQPullConsumer.shutdown(); + defaultMQAdminExt.shutdown(); + } + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/message/Store.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/message/Store.java new file mode 100644 index 000000000..53146a27c --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/message/Store.java @@ -0,0 +1,263 @@ +package com.alibaba.rocketmq.tools.command.message; + +import java.io.File; +import java.nio.ByteBuffer; +import java.util.Date; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; + +import com.alibaba.rocketmq.store.ConsumeQueue; +import com.alibaba.rocketmq.store.MapedFile; +import com.alibaba.rocketmq.store.MapedFileQueue; +import com.alibaba.rocketmq.store.SelectMapedBufferResult; +import com.alibaba.rocketmq.store.config.StorePathConfigHelper; + + +/** + * @auther lansheng.zj + */ +public class Store { + + // 每个消息对应的MAGIC CODE daa320a7 + public final static int MessageMagicCode = 0xAABBCCDD ^ 1880681586 + 8; + // 文件末尾空洞对应的MAGIC CODE cbd43194 + private final static int BlankMagicCode = 0xBBCCDDEE ^ 1880681586 + 8; + // 存储消息的队列 + private MapedFileQueue mapedFileQueue; + // ConsumeQueue集合 + private ConcurrentHashMap> consumeQueueTable; + + private String cStorePath; + private int cSize; + private String lStorePath; + private int lSize; + + + public Store(String cStorePath, int cSize, String lStorePath, int lSize) { + this.cStorePath = cStorePath; + this.cSize = cSize; + this.lStorePath = lStorePath; + this.lSize = lSize; + mapedFileQueue = new MapedFileQueue(cStorePath, cSize, null); + consumeQueueTable = + new ConcurrentHashMap>(); + } + + + public boolean load() { + boolean result = this.mapedFileQueue.load(); + System.out.println("load commit log " + (result ? "OK" : "Failed")); + if (result) { + result = loadConsumeQueue(); + } + System.out.println("load logics log " + (result ? "OK" : "Failed")); + return result; + } + + + private boolean loadConsumeQueue() { + File dirLogic = new File(StorePathConfigHelper.getStorePathConsumeQueue(lStorePath)); + File[] fileTopicList = dirLogic.listFiles(); + if (fileTopicList != null) { + // TOPIC 遍历 + for (File fileTopic : fileTopicList) { + String topic = fileTopic.getName(); + // TOPIC 下队列遍历 + File[] fileQueueIdList = fileTopic.listFiles(); + if (fileQueueIdList != null) { + for (File fileQueueId : fileQueueIdList) { + int queueId = Integer.parseInt(fileQueueId.getName()); + ConsumeQueue logic = new ConsumeQueue(// + topic,// + queueId,// + StorePathConfigHelper.getStorePathConsumeQueue(lStorePath),// + lSize,// + null); + this.putConsumeQueue(topic, queueId, logic); + if (!logic.load()) { + return false; + } + } + } + } + } + System.out.println("load logics queue all over, OK"); + return true; + } + + + private void putConsumeQueue(final String topic, final int queueId, final ConsumeQueue consumeQueue) { + ConcurrentHashMap map = this.consumeQueueTable.get(topic); + if (null == map) { + map = new ConcurrentHashMap(); + map.put(queueId, consumeQueue); + this.consumeQueueTable.put(topic, map); + } + else { + map.put(queueId, consumeQueue); + } + } + + + public ConsumeQueue findConsumeQueue(String topic, int queueId) { + ConcurrentHashMap map = consumeQueueTable.get(topic); + if (null == map) { + ConcurrentHashMap newMap = + new ConcurrentHashMap(128); + ConcurrentHashMap oldMap = consumeQueueTable.putIfAbsent(topic, newMap); + if (oldMap != null) { + map = oldMap; + } + else { + map = newMap; + } + } + ConsumeQueue logic = map.get(queueId); + if (null == logic) { + ConsumeQueue newLogic = new ConsumeQueue(// + topic,// + queueId,// + StorePathConfigHelper.getStorePathConsumeQueue(lStorePath),// + lSize,// + null); + ConsumeQueue oldLogic = map.putIfAbsent(queueId, newLogic); + if (oldLogic != null) { + logic = oldLogic; + } + else { + logic = newLogic; + } + } + return logic; + } + + + public void traval(boolean openAll) { + boolean success = true; + byte[] bytesContent = new byte[1024]; + List mapedFiles = this.mapedFileQueue.getMapedFiles(); + ALL: for (MapedFile mapedFile : mapedFiles) { + long startOffset = mapedFile.getFileFromOffset(); + int position = 0; + int msgCount = 0; + int errorCount = 0; + + System.out.println("start travel " + mapedFile.getFileName()); + long startTime = System.currentTimeMillis(); + ByteBuffer byteBuffer = mapedFile.sliceByteBuffer(); + while (byteBuffer.hasRemaining()) { + // 1 TOTALSIZE + int totalSize = byteBuffer.getInt(); + // 2 MAGICCODE + int magicCode = byteBuffer.getInt(); + if (BlankMagicCode == magicCode) { + position = byteBuffer.limit(); + break; + } + // 3 BODYCRC + int bodyCRC = byteBuffer.getInt(); + + // 4 QUEUEID + int queueId = byteBuffer.getInt(); + + // 5 FLAG + int flag = byteBuffer.getInt(); + flag = flag + 0; + + // 6 QUEUEOFFSET + long queueOffset = byteBuffer.getLong(); + + // 7 PHYSICALOFFSET + long physicOffset = byteBuffer.getLong(); + + // 8 SYSFLAG + int sysFlag = byteBuffer.getInt(); + + // 9 BORNTIMESTAMP + long bornTimeStamp = byteBuffer.getLong(); + bornTimeStamp = bornTimeStamp + 0; + + // 10 BORNHOST（IP+PORT） + byteBuffer.position(byteBuffer.position() + 8); + + // 11 STORETIMESTAMP + long storeTimestamp = byteBuffer.getLong(); + + // 12 STOREHOST（IP+PORT） + byteBuffer.position(byteBuffer.position() + 8); + + // 13 RECONSUMETIMES + int reconsumeTimes = byteBuffer.getInt(); + + // 14 Prepared Transaction Offset + long preparedTransactionOffset = byteBuffer.getLong(); + + // 15 BODY + int bodyLen = byteBuffer.getInt(); + if (bodyLen > 0) { + byteBuffer.position(byteBuffer.position() + bodyLen); + } + + // 16 TOPIC + byte topicLen = byteBuffer.get(); + byteBuffer.get(bytesContent, 0, topicLen); + String topic = new String(bytesContent, 0, topicLen); + + Date storeTime = new Date(storeTimestamp); + + // 计算出来当前消息的偏移量 + long currentPhyOffset = startOffset + position; + if (physicOffset != currentPhyOffset) { + System.out.println(storeTime + + " [fetal error] physicOffset != currentPhyOffset. position=" + position + + ", msgCount=" + msgCount + ", physicOffset=" + physicOffset + + ", currentPhyOffset=" + currentPhyOffset); + errorCount++; + if (!openAll) { + success = false; + break ALL; + } + } + + ConsumeQueue consumeQueue = findConsumeQueue(topic, queueId); + SelectMapedBufferResult smb = consumeQueue.getIndexBuffer(queueOffset); + try { + long offsetPy = smb.getByteBuffer().getLong(); + int sizePy = smb.getByteBuffer().getInt(); + if (physicOffset != offsetPy) { + System.out.println(storeTime + " [fetal error] physicOffset != offsetPy. position=" + + position + ", msgCount=" + msgCount + ", physicOffset=" + physicOffset + + ", offsetPy=" + offsetPy); + errorCount++; + if (!openAll) { + success = false; + break ALL; + } + } + if (totalSize != sizePy) { + System.out.println(storeTime + " [fetal error] totalSize != sizePy. position=" + + position + ", msgCount=" + msgCount + ", totalSize=" + totalSize + + ", sizePy=" + sizePy); + errorCount++; + if (!openAll) { + success = false; + break ALL; + } + } + } + finally { + smb.release(); + } + + msgCount++; + position += totalSize; + byteBuffer.position(position); + } + + System.out.println("end travel " + mapedFile.getFileName() + ", total msg=" + msgCount + + ", error count=" + errorCount + ", cost:" + (System.currentTimeMillis() - startTime)); + } + + System.out.println("travel " + (success ? "ok" : "fail")); + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/namesrv/DeleteKvConfigCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/namesrv/DeleteKvConfigCommand.java new file mode 100644 index 000000000..b88e47159 --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/namesrv/DeleteKvConfigCommand.java @@ -0,0 +1,81 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.tools.command.namesrv; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; + +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.tools.admin.DefaultMQAdminExt; +import com.alibaba.rocketmq.tools.command.SubCommand; + + +/** + * 删除 KV 配置信息 + * + * @author: manhong.yqd + * @since: 13-8-29 + */ +public class DeleteKvConfigCommand implements SubCommand { + @Override + public String commandName() { + return "deleteKvConfig"; + } + + + @Override + public String commandDesc() { + return "Delete KV config."; + } + + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("s", "namespace", true, "set the namespace"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("k", "key", true, "set the key name"); + opt.setRequired(true); + options.addOption(opt); + return options; + } + + + @Override + public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) { + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + try { + // namespace + String namespace = commandLine.getOptionValue('s').trim(); + // key name + String key = commandLine.getOptionValue('k').trim(); + + defaultMQAdminExt.start(); + defaultMQAdminExt.deleteKvConfig(namespace, key); + System.out.printf("delete kv config from namespace success.\n"); + return; + } + catch (Exception e) { + e.printStackTrace(); + } + finally { + defaultMQAdminExt.shutdown(); + } + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/namesrv/DeleteProjectGroupCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/namesrv/DeleteProjectGroupCommand.java new file mode 100644 index 000000000..f1286a365 --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/namesrv/DeleteProjectGroupCommand.java @@ -0,0 +1,91 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.tools.command.namesrv; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; + +import com.alibaba.rocketmq.common.namesrv.NamesrvUtil; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.srvutil.ServerUtil; +import com.alibaba.rocketmq.tools.admin.DefaultMQAdminExt; +import com.alibaba.rocketmq.tools.command.SubCommand; + + +/** + * 删除 project group 配置信息 + * + * @author: manhong.yqd + * @since: 13-8-29 + */ +public class DeleteProjectGroupCommand implements SubCommand { + @Override + public String commandName() { + return "deleteProjectGroup"; + } + + + @Override + public String commandDesc() { + return "Delete project group by server ip."; + } + + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("i", "ip", true, "set the server ip"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("p", "project", true, "set the project group"); + opt.setRequired(false); + options.addOption(opt); + return options; + } + + + @Override + public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) { + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + try { + String namespace = NamesrvUtil.NAMESPACE_PROJECT_CONFIG; + + if (commandLine.hasOption("i")) { + String ip = commandLine.getOptionValue('i').trim(); + defaultMQAdminExt.start(); + defaultMQAdminExt.deleteKvConfig(namespace, ip); + System.out.printf("delete project group from namespace by server ip success.\n"); + } + else if (commandLine.hasOption("p")) { + String project = commandLine.getOptionValue('p').trim(); + defaultMQAdminExt.start(); + defaultMQAdminExt.deleteIpsByProjectGroup(project); + System.out.printf("delete all server ip from namespace by project group success.\n"); + } + else { + ServerUtil.printCommandLineHelp("mqadmin " + this.commandName(), options); + } + } + catch (Exception e) { + e.printStackTrace(); + } + finally { + defaultMQAdminExt.shutdown(); + } + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/namesrv/GetProjectGroupCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/namesrv/GetProjectGroupCommand.java new file mode 100644 index 000000000..e3c21bc64 --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/namesrv/GetProjectGroupCommand.java @@ -0,0 +1,94 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.tools.command.namesrv; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; + +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.srvutil.ServerUtil; +import com.alibaba.rocketmq.tools.admin.DefaultMQAdminExt; +import com.alibaba.rocketmq.tools.command.SubCommand; + + +/** + * 取得 project group 配置信息 + * + * @author: manhong.yqd + * @since: 13-8-29 + */ +public class GetProjectGroupCommand implements SubCommand { + @Override + public String commandName() { + return "getProjectGroup"; + } + + + @Override + public String commandDesc() { + return "Get project group by server ip or project group name."; + } + + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("i", "ip", true, "set the server ip"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("p", "project", true, "set the project group"); + opt.setRequired(false); + options.addOption(opt); + return options; + } + + + @Override + public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) { + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + try { + if (commandLine.hasOption("i")) { + String ip = commandLine.getOptionValue('i').trim(); + defaultMQAdminExt.start(); + String project = defaultMQAdminExt.getProjectGroupByIp(ip); + System.out.printf("ip=%s, projectGroup=%s\n", ip, project); + } + else if (commandLine.hasOption("p")) { + String project = commandLine.getOptionValue('p').trim(); + defaultMQAdminExt.start(); + String ips = defaultMQAdminExt.getIpsByProjectGroup(project); + if (UtilAll.isBlank(ips)) { + System.out.printf("No ip in project group[%s]\n", project); + } + else { + System.out.printf("projectGroup=%s, ips=%s\n", project, ips); + } + } + else { + ServerUtil.printCommandLineHelp("mqadmin " + this.commandName(), options); + } + } + catch (Exception e) { + e.printStackTrace(); + } + finally { + defaultMQAdminExt.shutdown(); + } + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/namesrv/UpdateKvConfigCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/namesrv/UpdateKvConfigCommand.java new file mode 100644 index 000000000..9b1761990 --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/namesrv/UpdateKvConfigCommand.java @@ -0,0 +1,87 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.tools.command.namesrv; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; + +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.tools.admin.DefaultMQAdminExt; +import com.alibaba.rocketmq.tools.command.SubCommand; + + +/** + * 添加或者更新 KV 配置信息 + * + * @author: manhong.yqd + * @since: 13-8-29 + */ +public class UpdateKvConfigCommand implements SubCommand { + @Override + public String commandName() { + return "updateKvConfig"; + } + + + @Override + public String commandDesc() { + return "Create or update KV config."; + } + + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("s", "namespace", true, "set the namespace"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("k", "key", true, "set the key name"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("v", "value", true, "set the key value"); + opt.setRequired(true); + options.addOption(opt); + return options; + } + + + @Override + public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) { + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + try { + // namespace + String namespace = commandLine.getOptionValue('s').trim(); + // key name + String key = commandLine.getOptionValue('k').trim(); + // key name + String value = commandLine.getOptionValue('v').trim(); + + defaultMQAdminExt.start(); + defaultMQAdminExt.createAndUpdateKvConfig(namespace, key, value); + System.out.printf("create or update kv config to namespace success.\n"); + return; + } + catch (Exception e) { + e.printStackTrace(); + } + finally { + defaultMQAdminExt.shutdown(); + } + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/namesrv/UpdateProjectGroupCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/namesrv/UpdateProjectGroupCommand.java new file mode 100644 index 000000000..6a3b3c2f4 --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/namesrv/UpdateProjectGroupCommand.java @@ -0,0 +1,81 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.tools.command.namesrv; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; + +import com.alibaba.rocketmq.common.namesrv.NamesrvUtil; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.tools.admin.DefaultMQAdminExt; +import com.alibaba.rocketmq.tools.command.SubCommand; + + +/** + * 添加或者更新 project group 配置信息 + * + * @author: manhong.yqd + * @since: 13-8-29 + */ +public class UpdateProjectGroupCommand implements SubCommand { + @Override + public String commandName() { + return "updateProjectGroup"; + } + + + @Override + public String commandDesc() { + return "Create or update project group by server ip."; + } + + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("i", "ip", true, "set the server ip"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("p", "project", true, "set the project group"); + opt.setRequired(true); + options.addOption(opt); + return options; + } + + + @Override + public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) { + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + try { + String namespace = NamesrvUtil.NAMESPACE_PROJECT_CONFIG; + String ip = commandLine.getOptionValue('i').trim(); + String project = commandLine.getOptionValue('p').trim(); + + defaultMQAdminExt.start(); + defaultMQAdminExt.createAndUpdateKvConfig(namespace, ip, project); + System.out.printf("create or update kv config to namespace success.\n"); + return; + } + catch (Exception e) { + e.printStackTrace(); + } + finally { + defaultMQAdminExt.shutdown(); + } + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/namesrv/WipeWritePermSubCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/namesrv/WipeWritePermSubCommand.java new file mode 100644 index 000000000..dc40e965c --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/namesrv/WipeWritePermSubCommand.java @@ -0,0 +1,96 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.tools.command.namesrv; + +import java.util.List; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; + +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.tools.admin.DefaultMQAdminExt; +import com.alibaba.rocketmq.tools.command.SubCommand; + + +/** + * 从所有Name Server上清除特定Broker的先权限 + * + * @author shijia.wxr + * @since 2013-8-6 + */ +public class WipeWritePermSubCommand implements SubCommand { + + @Override + public String commandName() { + return "wipeWritePerm"; + } + + + @Override + public String commandDesc() { + return "Wipe write perm of broker in all name server"; + } + + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("b", "brokerName", true, "broker name"); + opt.setRequired(true); + options.addOption(opt); + return options; + } + + + @Override + public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) { + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + + try { + defaultMQAdminExt.start(); + String brokerName = commandLine.getOptionValue('b').trim(); + List namesrvList = defaultMQAdminExt.getNameServerAddressList(); + if (namesrvList != null) { + for (String namesrvAddr : namesrvList) { + try { + int wipeTopicCount = defaultMQAdminExt.wipeWritePermOfBroker(namesrvAddr, brokerName); + System.out.printf("wipe write perm of broker[%s] in name server[%s] OK, %d\n",// + brokerName,// + namesrvAddr,// + wipeTopicCount// + ); + } + catch (Exception e) { + System.out.printf("wipe write perm of broker[%s] in name server[%s] Failed\n",// + brokerName,// + namesrvAddr// + ); + + e.printStackTrace(); + } + } + } + } + catch (Exception e) { + e.printStackTrace(); + } + finally { + defaultMQAdminExt.shutdown(); + } + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/offset/CloneGroupOffsetCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/offset/CloneGroupOffsetCommand.java new file mode 100644 index 000000000..cb84a26ac --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/offset/CloneGroupOffsetCommand.java @@ -0,0 +1,100 @@ +package com.alibaba.rocketmq.tools.command.offset; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.PosixParser; + +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.srvutil.ServerUtil; +import com.alibaba.rocketmq.tools.admin.DefaultMQAdminExt; +import com.alibaba.rocketmq.tools.command.SubCommand; + + +/** + * 复制某一个 group 的 offset。 + * + * @author: manhong.yqd + * @since: 13-9-12 + */ +public class CloneGroupOffsetCommand implements SubCommand { + @Override + public String commandName() { + return "cloneGroupOffset"; + } + + + @Override + public String commandDesc() { + return "clone offset from other group."; + } + + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("s", "srcGroup", true, "set source consumer group"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("d", "destGroup", true, "set destination consumer group"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("t", "topic", true, "set the topic"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("o", "offline", true, "the group or the topic is offline"); + opt.setRequired(false); + options.addOption(opt); + + options.addOption(opt); + + return options; + } + + + @Override + public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) { + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + try { + String srcGroup = commandLine.getOptionValue("s").trim(); + String destGroup = commandLine.getOptionValue("d").trim(); + String topic = ""; + if (commandLine.hasOption('t')) { + topic = commandLine.getOptionValue("t").trim(); + } + + boolean isOffline = false; + if (commandLine.hasOption('o')) { + isOffline = Boolean.parseBoolean(commandLine.getOptionValue("o").trim()); + } + + defaultMQAdminExt.start(); + defaultMQAdminExt.cloneGroupOffset(srcGroup, destGroup, topic, isOffline); + System.out.printf("clone group offset success. srcGroup[%s], destGroup=[%s], topic[%s]", + srcGroup, destGroup, topic); + } + catch (Exception e) { + e.printStackTrace(); + } + finally { + defaultMQAdminExt.shutdown(); + } + } + + + public static void main(String[] args) { + System.setProperty(MixAll.NAMESRV_ADDR_PROPERTY, "127.0.0.1:9876"); + CloneGroupOffsetCommand cmd = new CloneGroupOffsetCommand(); + Options options = ServerUtil.buildCommandlineOptions(new Options()); + String[] subargs = + new String[] { "-t qatest_TopicTest", "-g qatest_consumer", "-s 1389098416742", "-f true" }; + final CommandLine commandLine = + ServerUtil.parseCmdLine("mqadmin " + cmd.commandName(), subargs, + cmd.buildCommandlineOptions(options), new PosixParser()); + cmd.execute(commandLine, options, null); + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/offset/GetConsumerStatusCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/offset/GetConsumerStatusCommand.java new file mode 100644 index 000000000..4698f6b3a --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/offset/GetConsumerStatusCommand.java @@ -0,0 +1,115 @@ +package com.alibaba.rocketmq.tools.command.offset; + +import java.util.Iterator; +import java.util.Map; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.PosixParser; + +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.srvutil.ServerUtil; +import com.alibaba.rocketmq.tools.admin.DefaultMQAdminExt; +import com.alibaba.rocketmq.tools.command.SubCommand; + + +/** + * 根据时间来设置消费进度，设置之前要关闭这个订阅组的所有consumer，设置完再启动，方可生效。 + * + * @author: manhong.yqd + * @since: 13-9-12 + */ +public class GetConsumerStatusCommand implements SubCommand { + @Override + public String commandName() { + return "getConsumerStatus"; + } + + + @Override + public String commandDesc() { + return "get consumer status from client."; + } + + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("g", "group", true, "set the consumer group"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("t", "topic", true, "set the topic"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("i", "originClientId", true, "set the consumer clientId"); + opt.setRequired(false); + options.addOption(opt); + + return options; + } + + + @Override + public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) { + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + try { + String group = commandLine.getOptionValue("g").trim(); + String topic = commandLine.getOptionValue("t").trim(); + String originClientId = ""; + if (commandLine.hasOption("i")) { + originClientId = commandLine.getOptionValue("i").trim(); + } + defaultMQAdminExt.start(); + + Map> consumerStatusTable = + defaultMQAdminExt.getConsumeStatus(topic, group, originClientId); + System.out.printf("get consumer status from client. group=%s, topic=%s, originClientId=%s\n", + group, topic, originClientId); + + System.out.printf("%-50s %-15s %-15s %-20s\n",// + "#clientId",// + "#brokerName", // + "#queueId",// + "#offset"); + + Iterator clientIterator = consumerStatusTable.keySet().iterator(); + while (clientIterator.hasNext()) { + String clientId = clientIterator.next(); + Map mqTable = consumerStatusTable.get(clientId); + Iterator mqIterator = mqTable.keySet().iterator(); + while (mqIterator.hasNext()) { + MessageQueue mq = mqIterator.next(); + System.out.printf("%-50s %-15s %-15d %-20d\n",// + UtilAll.frontStringAtLeast(clientId, 50),// + mq.getBrokerName(),// + mq.getQueueId(),// + mqTable.get(mq)); + } + } + } + catch (Exception e) { + e.printStackTrace(); + } + finally { + defaultMQAdminExt.shutdown(); + } + } + + + public static void main(String[] args) { + System.setProperty(MixAll.NAMESRV_ADDR_PROPERTY, "127.0.0.1:9876"); + GetConsumerStatusCommand cmd = new GetConsumerStatusCommand(); + Options options = ServerUtil.buildCommandlineOptions(new Options()); + String[] subargs = new String[] { "-t qatest_TopicTest", "-g qatest_consumer_broadcast" }; + final CommandLine commandLine = + ServerUtil.parseCmdLine("mqadmin " + cmd.commandName(), subargs, + cmd.buildCommandlineOptions(options), new PosixParser()); + cmd.execute(commandLine, options, null); + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/offset/ResetOffsetByTimeCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/offset/ResetOffsetByTimeCommand.java new file mode 100644 index 000000000..4d667414f --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/offset/ResetOffsetByTimeCommand.java @@ -0,0 +1,140 @@ +package com.alibaba.rocketmq.tools.command.offset; + +import java.util.Iterator; +import java.util.Map; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.PosixParser; + +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.common.protocol.ResponseCode; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.srvutil.ServerUtil; +import com.alibaba.rocketmq.tools.admin.DefaultMQAdminExt; +import com.alibaba.rocketmq.tools.command.SubCommand; + + +/** + * 根据时间设置消费进度，客户端无需重启。 + * + * @author: manhong.yqd + * @since: 13-9-12 + */ +public class ResetOffsetByTimeCommand implements SubCommand { + @Override + public String commandName() { + return "resetOffsetByTime"; + } + + + @Override + public String commandDesc() { + return "Reset consumer offset by timestamp(without client restart)."; + } + + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("g", "group", true, "set the consumer group"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("t", "topic", true, "set the topic"); + opt.setRequired(true); + options.addOption(opt); + + opt = + new Option("s", "timestamp", true, + "set the timestamp[currentTimeMillis|yyyy-MM-dd#HH:mm:ss:SSS]"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("f", "force", true, "set the force rollback by timestamp switch[true|false]"); + opt.setRequired(false); + options.addOption(opt); + return options; + } + + + @Override + public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) { + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + try { + String group = commandLine.getOptionValue("g").trim(); + String topic = commandLine.getOptionValue("t").trim(); + String timeStampStr = commandLine.getOptionValue("s").trim(); + long timestamp = 0; + try { + // 直接输入 long 类型的 timestamp + timestamp = Long.valueOf(timeStampStr); + } + catch (NumberFormatException e) { + // 输入的为日期格式，精确到毫秒 + timestamp = UtilAll.parseDate(timeStampStr, UtilAll.yyyy_MM_dd_HH_mm_ss_SSS).getTime(); + } + + boolean force = true; + if (commandLine.hasOption('f')) { + force = Boolean.valueOf(commandLine.getOptionValue("f").trim()); + } + + defaultMQAdminExt.start(); + Map offsetTable; + try { + offsetTable = defaultMQAdminExt.resetOffsetByTimestamp(topic, group, timestamp, force); + } + catch (MQClientException e) { + if (ResponseCode.CONSUMER_NOT_ONLINE == e.getResponseCode()) { + ResetOffsetByTimeOldCommand.resetOffset(defaultMQAdminExt, group, topic, timestamp, + force, timeStampStr); + return; + } + throw e; + } + + System.out + .printf( + "rollback consumer offset by specified group[%s], topic[%s], force[%s], timestamp(string)[%s], timestamp(long)[%s]\n", + group, topic, force, timeStampStr, timestamp); + + System.out.printf("%-40s %-40s %-40s\n",// + "#brokerName",// + "#queueId",// + "#offset"); + + Iterator> iterator = offsetTable.entrySet().iterator(); + while (iterator.hasNext()) { + Map.Entry entry = iterator.next(); + System.out.printf("%-40s %-40d %-40d\n",// + UtilAll.frontStringAtLeast(entry.getKey().getBrokerName(), 32),// + entry.getKey().getQueueId(),// + entry.getValue()); + } + } + catch (Exception e) { + e.printStackTrace(); + } + finally { + defaultMQAdminExt.shutdown(); + } + } + + + public static void main(String[] args) { + System.setProperty(MixAll.NAMESRV_ADDR_PROPERTY, "127.0.0.1:9876"); + ResetOffsetByTimeCommand cmd = new ResetOffsetByTimeCommand(); + Options options = ServerUtil.buildCommandlineOptions(new Options()); + String[] subargs = + new String[] { "-t qatest_TopicTest", "-g qatest_consumer", "-s 1389098416742", "-f true" }; + final CommandLine commandLine = + ServerUtil.parseCmdLine("mqadmin " + cmd.commandName(), subargs, + cmd.buildCommandlineOptions(options), new PosixParser()); + cmd.execute(commandLine, options, null); + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/offset/ResetOffsetByTimeOldCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/offset/ResetOffsetByTimeOldCommand.java new file mode 100644 index 000000000..1d42e56c2 --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/offset/ResetOffsetByTimeOldCommand.java @@ -0,0 +1,126 @@ +package com.alibaba.rocketmq.tools.command.offset; + +import java.util.List; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; + +import com.alibaba.rocketmq.client.exception.MQBrokerException; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.admin.RollbackStats; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.remoting.exception.RemotingException; +import com.alibaba.rocketmq.tools.admin.DefaultMQAdminExt; +import com.alibaba.rocketmq.tools.command.SubCommand; + + +/** + * 根据时间来设置消费进度，设置之前要关闭这个订阅组的所有consumer，设置完再启动，方可生效。 + * + * @author: manhong.yqd + * @since: 13-9-12 + */ +public class ResetOffsetByTimeOldCommand implements SubCommand { + @Override + public String commandName() { + return "resetOffsetByTimeOld"; + } + + + @Override + public String commandDesc() { + return "Reset consumer offset by timestamp(execute this command required client restart)."; + } + + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("g", "group", true, "set the consumer group"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("t", "topic", true, "set the topic"); + opt.setRequired(true); + options.addOption(opt); + + opt = + new Option("s", "timestamp", true, + "set the timestamp[currentTimeMillis|yyyy-MM-dd#HH:mm:ss:SSS]"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("f", "force", true, "set the force rollback by timestamp switch[true|false]"); + opt.setRequired(false); + options.addOption(opt); + return options; + } + + + public static void resetOffset(DefaultMQAdminExt defaultMQAdminExt, String consumerGroup, String topic, + long timestamp, boolean force, String timeStampStr) throws RemotingException, MQBrokerException, + InterruptedException, MQClientException { + List rollbackStatsList = + defaultMQAdminExt.resetOffsetByTimestampOld(consumerGroup, topic, timestamp, force); + System.out + .printf( + "rollback consumer offset by specified consumerGroup[%s], topic[%s], force[%s], timestamp(string)[%s], timestamp(long)[%s]\n", + consumerGroup, topic, force, timeStampStr, timestamp); + + System.out.printf("%-20s %-20s %-20s %-20s %-20s %-20s\n",// + "#brokerName",// + "#queueId",// + "#brokerOffset",// + "#consumerOffset",// + "#timestampOffset",// + "#rollbackOffset" // + ); + + for (RollbackStats rollbackStats : rollbackStatsList) { + System.out.printf("%-20s %-20d %-20d %-20d %-20d %-20d\n",// + UtilAll.frontStringAtLeast(rollbackStats.getBrokerName(), 32),// + rollbackStats.getQueueId(),// + rollbackStats.getBrokerOffset(),// + rollbackStats.getConsumerOffset(),// + rollbackStats.getTimestampOffset(),// + rollbackStats.getRollbackOffset() // + ); + } + } + + + @Override + public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) { + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + try { + String consumerGroup = commandLine.getOptionValue("g").trim(); + String topic = commandLine.getOptionValue("t").trim(); + String timeStampStr = commandLine.getOptionValue("s").trim(); + long timestamp = 0; + try { + // 直接输入 long 类型的 timestamp + timestamp = Long.valueOf(timeStampStr); + } + catch (NumberFormatException e) { + // 输入的为日期格式，精确到毫秒 + timestamp = UtilAll.parseDate(timeStampStr, UtilAll.yyyy_MM_dd_HH_mm_ss_SSS).getTime(); + } + + boolean force = true; + if (commandLine.hasOption('f')) { + force = Boolean.valueOf(commandLine.getOptionValue("f").trim()); + } + + defaultMQAdminExt.start(); + resetOffset(defaultMQAdminExt, consumerGroup, topic, timestamp, force, timeStampStr); + } + catch (Exception e) { + e.printStackTrace(); + } + finally { + defaultMQAdminExt.shutdown(); + } + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/topic/DeleteTopicSubCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/topic/DeleteTopicSubCommand.java new file mode 100644 index 000000000..95e393d1a --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/topic/DeleteTopicSubCommand.java @@ -0,0 +1,115 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.tools.command.topic; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; + +import com.alibaba.rocketmq.client.exception.MQBrokerException; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.remoting.exception.RemotingException; +import com.alibaba.rocketmq.srvutil.ServerUtil; +import com.alibaba.rocketmq.tools.admin.DefaultMQAdminExt; +import com.alibaba.rocketmq.tools.command.CommandUtil; +import com.alibaba.rocketmq.tools.command.SubCommand; + + +/** + * 删除Topic配置命令 + * + * @author manhong.yqd + * @since 2013-8-21 + */ +public class DeleteTopicSubCommand implements SubCommand { + @Override + public String commandName() { + return "deleteTopic"; + } + + + @Override + public String commandDesc() { + return "Delete topic from broker and NameServer."; + } + + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("t", "topic", true, "topic name"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("c", "clusterName", true, "delete topic from which cluster"); + opt.setRequired(true); + options.addOption(opt); + + return options; + } + + + public static void deleteTopic(final DefaultMQAdminExt adminExt,// + final String clusterName,// + final String topic// + ) throws InterruptedException, MQBrokerException, RemotingException, MQClientException { + // 删除 broker 上的 topic 信息 + Set masterSet = CommandUtil.fetchMasterAddrByClusterName(adminExt, clusterName); + adminExt.deleteTopicInBroker(masterSet, topic); + System.out.printf("delete topic [%s] from cluster [%s] success.\n", topic, clusterName); + + // 删除 NameServer 上的 topic 信息 + Set nameServerSet = null; + if (adminExt.getNamesrvAddr() != null) { + String[] ns = adminExt.getNamesrvAddr().trim().split(";"); + nameServerSet = new HashSet(Arrays.asList(ns)); + } + + // 删除 NameServer 上的 topic 信息 + adminExt.deleteTopicInNameServer(nameServerSet, topic); + System.out.printf("delete topic [%s] from NameServer success.\n", topic); + } + + + @Override + public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) { + DefaultMQAdminExt adminExt = new DefaultMQAdminExt(rpcHook); + adminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + try { + String topic = commandLine.getOptionValue('t').trim(); + + if (commandLine.hasOption('c')) { + String clusterName = commandLine.getOptionValue('c').trim(); + + adminExt.start(); + deleteTopic(adminExt, clusterName, topic); + return; + } + + ServerUtil.printCommandLineHelp("mqadmin " + this.commandName(), options); + } + catch (Exception e) { + e.printStackTrace(); + } + finally { + adminExt.shutdown(); + } + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/topic/TopicListSubCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/topic/TopicListSubCommand.java new file mode 100644 index 000000000..5a9a7e563 --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/topic/TopicListSubCommand.java @@ -0,0 +1,155 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.tools.command.topic; + +import java.util.Iterator; +import java.util.Map.Entry; +import java.util.Set; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; + +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.protocol.body.ClusterInfo; +import com.alibaba.rocketmq.common.protocol.body.GroupList; +import com.alibaba.rocketmq.common.protocol.body.TopicList; +import com.alibaba.rocketmq.common.protocol.route.BrokerData; +import com.alibaba.rocketmq.common.protocol.route.TopicRouteData; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.remoting.exception.RemotingException; +import com.alibaba.rocketmq.tools.admin.DefaultMQAdminExt; +import com.alibaba.rocketmq.tools.command.SubCommand; + + +/** + * 查看Topic统计信息，包括offset、最后更新时间 + * + * @author shijia.wxr + * @since 2013-8-3 + */ +public class TopicListSubCommand implements SubCommand { + + @Override + public String commandName() { + return "topicList"; + } + + + @Override + public String commandDesc() { + return "Fetch all topic list from name server"; + } + + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("c", "clusterModel", false, "clusterModel"); + opt.setRequired(false); + options.addOption(opt); + + return options; + } + + + private String findTopicBelongToWhichCluster(final String topic, final ClusterInfo clusterInfo, + final DefaultMQAdminExt defaultMQAdminExt) throws RemotingException, MQClientException, + InterruptedException { + TopicRouteData topicRouteData = defaultMQAdminExt.examineTopicRouteInfo(topic); + + BrokerData brokerData = topicRouteData.getBrokerDatas().get(0); + + String brokerName = brokerData.getBrokerName(); + + Iterator>> it = clusterInfo.getClusterAddrTable().entrySet().iterator(); + while (it.hasNext()) { + Entry> next = it.next(); + if (next.getValue().contains(brokerName)) { + return next.getKey(); + } + } + + return null; + } + + + @Override + public void execute(final CommandLine commandLine, final Options options, RPCHook rpcHook) { + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + + try { + defaultMQAdminExt.start(); + + if (commandLine.hasOption('c')) { + ClusterInfo clusterInfo = defaultMQAdminExt.examineBrokerClusterInfo(); + + System.out.printf("%-20s %-48s %-48s\n",// + "#Cluster Name",// + "#Topic",// + "#Consumer Group"// + ); + + TopicList topicList = defaultMQAdminExt.fetchAllTopicList(); + for (String topic : topicList.getTopicList()) { + if (topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX) + || topic.startsWith(MixAll.DLQ_GROUP_TOPIC_PREFIX)) { + continue; + } + + String clusterName = ""; + GroupList groupList = new GroupList(); + + try { + clusterName = + this.findTopicBelongToWhichCluster(topic, clusterInfo, defaultMQAdminExt); + groupList = defaultMQAdminExt.queryTopicConsumeByWho(topic); + } + catch (Exception e) { + } + + if (null == groupList || groupList.getGroupList().isEmpty()) { + groupList = new GroupList(); + groupList.getGroupList().add(""); + } + + for (String group : groupList.getGroupList()) { + System.out.printf("%-20s %-48s %-48s\n",// + UtilAll.frontStringAtLeast(clusterName, 20),// + UtilAll.frontStringAtLeast(topic, 48),// + UtilAll.frontStringAtLeast(group, 48)// + ); + } + } + } + else { + TopicList topicList = defaultMQAdminExt.fetchAllTopicList(); + for (String topic : topicList.getTopicList()) { + System.out.println(topic); + } + } + } + catch (Exception e) { + e.printStackTrace(); + } + finally { + defaultMQAdminExt.shutdown(); + } + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/topic/TopicRouteSubCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/topic/TopicRouteSubCommand.java new file mode 100644 index 000000000..2b6254c21 --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/topic/TopicRouteSubCommand.java @@ -0,0 +1,79 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.tools.command.topic; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; + +import com.alibaba.rocketmq.common.protocol.route.TopicRouteData; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.tools.admin.DefaultMQAdminExt; +import com.alibaba.rocketmq.tools.command.SubCommand; + + +/** + * 查看Topic路由信息 + * + * @author shijia.wxr + * @since 2013-8-3 + */ +public class TopicRouteSubCommand implements SubCommand { + + @Override + public String commandName() { + return "topicRoute"; + } + + + @Override + public String commandDesc() { + return "Examine topic route info"; + } + + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("t", "topic", true, "topic name"); + opt.setRequired(true); + options.addOption(opt); + + return options; + } + + + @Override + public void execute(final CommandLine commandLine, final Options options, RPCHook rpcHook) { + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + + try { + defaultMQAdminExt.start(); + + String topic = commandLine.getOptionValue('t').trim(); + TopicRouteData topicRouteData = defaultMQAdminExt.examineTopicRouteInfo(topic); + String json = topicRouteData.toJson(true); + System.out.println(json); + } + catch (Exception e) { + e.printStackTrace(); + } + finally { + defaultMQAdminExt.shutdown(); + } + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/topic/TopicStatusSubCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/topic/TopicStatusSubCommand.java new file mode 100644 index 000000000..9fe6c3061 --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/topic/TopicStatusSubCommand.java @@ -0,0 +1,113 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.tools.command.topic; + +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; + +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.admin.TopicOffset; +import com.alibaba.rocketmq.common.admin.TopicStatsTable; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.tools.admin.DefaultMQAdminExt; +import com.alibaba.rocketmq.tools.command.SubCommand; + + +/** + * 查看Topic统计信息，包括offset、最后更新时间 + * + * @author shijia.wxr + * @since 2013-8-3 + */ +public class TopicStatusSubCommand implements SubCommand { + + @Override + public String commandName() { + return "topicStatus"; + } + + + @Override + public String commandDesc() { + return "Examine topic Status info"; + } + + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("t", "topic", true, "topic name"); + opt.setRequired(true); + options.addOption(opt); + + return options; + } + + + @Override + public void execute(final CommandLine commandLine, final Options options, RPCHook rpcHook) { + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + + try { + defaultMQAdminExt.start(); + + String topic = commandLine.getOptionValue('t').trim(); + TopicStatsTable topicStatsTable = defaultMQAdminExt.examineTopicStats(topic); + + List mqList = new LinkedList(); + mqList.addAll(topicStatsTable.getOffsetTable().keySet()); + Collections.sort(mqList); + + System.out.printf("%-32s %-4s %-20s %-20s %s\n",// + "#Broker Name",// + "#QID",// + "#Min Offset",// + "#Max Offset",// + "#Last Updated" // + ); + + for (MessageQueue mq : mqList) { + TopicOffset topicOffset = topicStatsTable.getOffsetTable().get(mq); + + String humanTimestamp = ""; + if (topicOffset.getLastUpdateTimestamp() > 0) { + humanTimestamp = UtilAll.timeMillisToHumanString2(topicOffset.getLastUpdateTimestamp()); + } + + System.out.printf("%-32s %-4d %-20d %-20d %s\n",// + UtilAll.frontStringAtLeast(mq.getBrokerName(), 32),// + mq.getQueueId(),// + topicOffset.getMinOffset(),// + topicOffset.getMaxOffset(),// + humanTimestamp // + ); + } + } + catch (Exception e) { + e.printStackTrace(); + } + finally { + defaultMQAdminExt.shutdown(); + } + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/topic/UpdateOrderConfCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/topic/UpdateOrderConfCommand.java new file mode 100644 index 000000000..4f96704d3 --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/topic/UpdateOrderConfCommand.java @@ -0,0 +1,120 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.tools.command.topic; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; + +import com.alibaba.rocketmq.common.UtilAll; +import com.alibaba.rocketmq.common.namesrv.NamesrvUtil; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.srvutil.ServerUtil; +import com.alibaba.rocketmq.tools.admin.DefaultMQAdminExt; +import com.alibaba.rocketmq.tools.command.SubCommand; + + +/** + * 创建、修改、删除顺序消息 Topic 的分区配置命令 + * + * @author manhong.yqd + * @since 2014-2-20 + */ +public class UpdateOrderConfCommand implements SubCommand { + + @Override + public String commandName() { + return "updateOrderConf"; + } + + + @Override + public String commandDesc() { + return "Create or update or delete order conf"; + } + + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("t", "topic", true, "topic name"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("v", "orderConf", true, "set order conf [eg. brokerName1:num;brokerName2:num]"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("m", "method", true, "option type [eg. put|get|delete"); + opt.setRequired(true); + options.addOption(opt); + + return options; + } + + + @Override + public void execute(final CommandLine commandLine, final Options options, RPCHook rpcHook) { + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + + try { + String topic = commandLine.getOptionValue('t').trim(); + String type = commandLine.getOptionValue('m').trim(); + + if ("get".equals(type)) { + // 获取顺序消息 + defaultMQAdminExt.start(); + String orderConf = + defaultMQAdminExt.getKVConfig(NamesrvUtil.NAMESPACE_ORDER_TOPIC_CONFIG, topic); + System.out.printf("get orderConf success. topic=[%s], orderConf=[%s] ", topic, orderConf); + + return; + } + else if ("put".equals(type)) { + // 更新顺序消息 + defaultMQAdminExt.start(); + String orderConf = ""; + if (commandLine.hasOption('v')) { + orderConf = commandLine.getOptionValue('v').trim(); + } + if (UtilAll.isBlank(orderConf)) { + throw new Exception("please set orderConf with option -v."); + } + + defaultMQAdminExt.createOrUpdateOrderConf(topic, orderConf, true); + System.out.printf("update orderConf success. topic=[%s], orderConf=[%s]", topic, + orderConf.toString()); + return; + } + else if ("delete".equals(type)) { + // 删除顺序消息 + defaultMQAdminExt.start(); + defaultMQAdminExt.deleteKvConfig(NamesrvUtil.NAMESPACE_ORDER_TOPIC_CONFIG, topic); + System.out.printf("delete orderConf success. topic=[%s]", topic); + + return; + } + + ServerUtil.printCommandLineHelp("mqadmin " + this.commandName(), options); + } + catch (Exception e) { + e.printStackTrace(); + } + finally { + defaultMQAdminExt.shutdown(); + } + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/topic/UpdateTopicSubCommand.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/topic/UpdateTopicSubCommand.java new file mode 100644 index 000000000..ea2511703 --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/command/topic/UpdateTopicSubCommand.java @@ -0,0 +1,201 @@ +/** + * Copyright (C) 2010-2013 Alibaba Group Holding Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alibaba.rocketmq.tools.command.topic; + +import java.util.Set; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; + +import com.alibaba.rocketmq.common.TopicConfig; +import com.alibaba.rocketmq.common.sysflag.TopicSysFlag; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.srvutil.ServerUtil; +import com.alibaba.rocketmq.tools.admin.DefaultMQAdminExt; +import com.alibaba.rocketmq.tools.command.CommandUtil; +import com.alibaba.rocketmq.tools.command.SubCommand; + + +/** + * 修改、创建Topic配置命令 + * + * @author shijia.wxr + * @since 2013-7-21 + */ +public class UpdateTopicSubCommand implements SubCommand { + + @Override + public String commandName() { + return "updateTopic"; + } + + + @Override + public String commandDesc() { + return "Update or create topic"; + } + + + @Override + public Options buildCommandlineOptions(Options options) { + Option opt = new Option("b", "brokerAddr", true, "create topic to which broker"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("c", "clusterName", true, "create topic to which cluster"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("t", "topic", true, "topic name"); + opt.setRequired(true); + options.addOption(opt); + + opt = new Option("r", "readQueueNums", true, "set read queue nums"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("w", "writeQueueNums", true, "set write queue nums"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("p", "perm", true, "set topic's permission(2|4|6), intro[2:R; 4:W; 6:RW]"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("o", "order", true, "set topic's order(true|false"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("u", "unit", true, "is unit topic (true|false"); + opt.setRequired(false); + options.addOption(opt); + + opt = new Option("s", "hasUnitSub", true, "has unit sub (true|false"); + opt.setRequired(false); + options.addOption(opt); + + return options; + } + + + @Override + public void execute(final CommandLine commandLine, final Options options, RPCHook rpcHook) { + DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + defaultMQAdminExt.setInstanceName(Long.toString(System.currentTimeMillis())); + + try { + TopicConfig topicConfig = new TopicConfig(); + topicConfig.setReadQueueNums(8); + topicConfig.setWriteQueueNums(8); + topicConfig.setTopicName(commandLine.getOptionValue('t').trim()); + + // readQueueNums + if (commandLine.hasOption('r')) { + topicConfig.setReadQueueNums(Integer.parseInt(commandLine.getOptionValue('r').trim())); + } + + // writeQueueNums + if (commandLine.hasOption('w')) { + topicConfig.setWriteQueueNums(Integer.parseInt(commandLine.getOptionValue('w').trim())); + } + + // perm + if (commandLine.hasOption('p')) { + topicConfig.setPerm(Integer.parseInt(commandLine.getOptionValue('p').trim())); + } + + boolean isUnit = false; + if (commandLine.hasOption('u')) { + isUnit = Boolean.parseBoolean(commandLine.getOptionValue('u').trim()); + } + + boolean isCenterSync = false; + if (commandLine.hasOption('s')) { + isCenterSync = Boolean.parseBoolean(commandLine.getOptionValue('s').trim()); + } + + int topicCenterSync = TopicSysFlag.buildSysFlag(isUnit, isCenterSync); + topicConfig.setTopicSysFlag(topicCenterSync); + + boolean isOrder = false; + if (commandLine.hasOption('o')) { + isOrder = Boolean.parseBoolean(commandLine.getOptionValue('o').trim()); + } + topicConfig.setOrder(isOrder); + + if (commandLine.hasOption('b')) { + String addr = commandLine.getOptionValue('b').trim(); + + defaultMQAdminExt.start(); + defaultMQAdminExt.createAndUpdateTopicConfig(addr, topicConfig); + + if (isOrder) { + // 注册顺序消息到 nameserver + String brokerName = CommandUtil.fetchBrokerNameByAddr(defaultMQAdminExt, addr); + String orderConf = brokerName + ":" + topicConfig.getWriteQueueNums(); + defaultMQAdminExt.createOrUpdateOrderConf(topicConfig.getTopicName(), orderConf, false); + System.out.println(String.format("set broker orderConf. isOrder=%s, orderConf=[%s]", + isOrder, orderConf.toString())); + } + System.out.printf("create topic to %s success.\n", addr); + System.out.println(topicConfig); + return; + + } + else if (commandLine.hasOption('c')) { + String clusterName = commandLine.getOptionValue('c').trim(); + + defaultMQAdminExt.start(); + + Set masterSet = + CommandUtil.fetchMasterAddrByClusterName(defaultMQAdminExt, clusterName); + for (String addr : masterSet) { + defaultMQAdminExt.createAndUpdateTopicConfig(addr, topicConfig); + System.out.printf("create topic to %s success.\n", addr); + } + + if (isOrder) { + // 注册顺序消息到 nameserver + Set brokerNameSet = + CommandUtil.fetchBrokerNameByClusterName(defaultMQAdminExt, clusterName); + StringBuilder orderConf = new StringBuilder(); + String splitor = ""; + for (String s : brokerNameSet) { + orderConf.append(splitor).append(s).append(":") + .append(topicConfig.getWriteQueueNums()); + splitor = ";"; + } + defaultMQAdminExt.createOrUpdateOrderConf(topicConfig.getTopicName(), + orderConf.toString(), true); + System.out.println(String.format("set cluster orderConf. isOrder=%s, orderConf=[%s]", + isOrder, orderConf.toString())); + } + + System.out.println(topicConfig); + return; + } + + ServerUtil.printCommandLineHelp("mqadmin " + this.commandName(), options); + } + catch (Exception e) { + e.printStackTrace(); + } + finally { + defaultMQAdminExt.shutdown(); + } + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/monitor/DefaultMonitorListener.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/monitor/DefaultMonitorListener.java new file mode 100644 index 000000000..375fc1efe --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/monitor/DefaultMonitorListener.java @@ -0,0 +1,83 @@ +package com.alibaba.rocketmq.tools.monitor; + +import java.util.Iterator; +import java.util.Map.Entry; +import java.util.TreeMap; + +import org.slf4j.Logger; + +import com.alibaba.rocketmq.client.log.ClientLogger; +import com.alibaba.rocketmq.common.protocol.body.ConsumerRunningInfo; + + +public class DefaultMonitorListener implements MonitorListener { + private final Logger log = ClientLogger.getLog(); + + private final static String LogPrefix = "[MONITOR] "; + + private final static String LogNotify = LogPrefix + " [NOTIFY] "; + + + public DefaultMonitorListener() { + } + + + @Override + public void beginRound() { + log.info(LogPrefix + "=========================================beginRound"); + } + + + @Override + public void reportUndoneMsgs(UndoneMsgs undoneMsgs) { + log.info(String.format(LogPrefix + "reportUndoneMsgs: %s", undoneMsgs)); + } + + + @Override + public void reportFailedMsgs(FailedMsgs failedMsgs) { + log.info(String.format(LogPrefix + "reportFailedMsgs: %s", failedMsgs)); + } + + + @Override + public void reportDeleteMsgsEvent(DeleteMsgsEvent deleteMsgsEvent) { + log.info(String.format(LogPrefix + "reportDeleteMsgsEvent: %s", deleteMsgsEvent)); + } + + + @Override + public void reportConsumerRunningInfo(TreeMap criTable) { + // 分析订阅关系 + { + boolean result = ConsumerRunningInfo.analyzeSubscription(criTable); + if (!result) { + log.info(String.format(LogNotify + + "reportConsumerRunningInfo: ConsumerGroup: %s, Subscription different", criTable + .firstEntry().getValue().getProperties().getProperty("consumerGroup"))); + } + } + + // 分析顺序消息 + { + Iterator> it = criTable.entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + String result = ConsumerRunningInfo.analyzeProcessQueue(next.getKey(), next.getValue()); + if (result != null && !result.isEmpty()) { + log.info(String.format(LogNotify + + "reportConsumerRunningInfo: ConsumerGroup: %s, ClientId: %s, %s", // + criTable.firstEntry().getValue().getProperties().getProperty("consumerGroup"),// + next.getKey(),// + result)); + } + } + } + } + + + @Override + public void endRound() { + log.info(LogPrefix + "=========================================endRound"); + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/monitor/DeleteMsgsEvent.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/monitor/DeleteMsgsEvent.java new file mode 100644 index 000000000..fc73c0160 --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/monitor/DeleteMsgsEvent.java @@ -0,0 +1,36 @@ +package com.alibaba.rocketmq.tools.monitor; + +import com.alibaba.rocketmq.common.protocol.topic.OffsetMovedEvent; + + +public class DeleteMsgsEvent { + private OffsetMovedEvent offsetMovedEvent; + private long eventTimestamp; + + + public OffsetMovedEvent getOffsetMovedEvent() { + return offsetMovedEvent; + } + + + public void setOffsetMovedEvent(OffsetMovedEvent offsetMovedEvent) { + this.offsetMovedEvent = offsetMovedEvent; + } + + + public long getEventTimestamp() { + return eventTimestamp; + } + + + public void setEventTimestamp(long eventTimestamp) { + this.eventTimestamp = eventTimestamp; + } + + + @Override + public String toString() { + return "DeleteMsgsEvent [offsetMovedEvent=" + offsetMovedEvent + ", eventTimestamp=" + eventTimestamp + + "]"; + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/monitor/FailedMsgs.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/monitor/FailedMsgs.java new file mode 100644 index 000000000..a159561a5 --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/monitor/FailedMsgs.java @@ -0,0 +1,44 @@ +package com.alibaba.rocketmq.tools.monitor; + +public class FailedMsgs { + private String consumerGroup; + private String topic; + private long failedMsgsTotalRecently; + + + public String getConsumerGroup() { + return consumerGroup; + } + + + public void setConsumerGroup(String consumerGroup) { + this.consumerGroup = consumerGroup; + } + + + public String getTopic() { + return topic; + } + + + public void setTopic(String topic) { + this.topic = topic; + } + + + public long getFailedMsgsTotalRecently() { + return failedMsgsTotalRecently; + } + + + public void setFailedMsgsTotalRecently(long failedMsgsTotalRecently) { + this.failedMsgsTotalRecently = failedMsgsTotalRecently; + } + + + @Override + public String toString() { + return "FailedMsgs [consumerGroup=" + consumerGroup + ", topic=" + topic + + ", failedMsgsTotalRecently=" + failedMsgsTotalRecently + "]"; + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/monitor/MonitorConfig.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/monitor/MonitorConfig.java new file mode 100644 index 000000000..4887b1a25 --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/monitor/MonitorConfig.java @@ -0,0 +1,31 @@ +package com.alibaba.rocketmq.tools.monitor; + +import com.alibaba.rocketmq.common.MixAll; + + +public class MonitorConfig { + private String namesrvAddr = System.getProperty(MixAll.NAMESRV_ADDR_PROPERTY, + System.getenv(MixAll.NAMESRV_ADDR_ENV)); + // 监控一轮间隔时间 + private int roundInterval = 1000 * 60; + + + public String getNamesrvAddr() { + return namesrvAddr; + } + + + public void setNamesrvAddr(String namesrvAddr) { + this.namesrvAddr = namesrvAddr; + } + + + public int getRoundInterval() { + return roundInterval; + } + + + public void setRoundInterval(int roundInterval) { + this.roundInterval = roundInterval; + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/monitor/MonitorListener.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/monitor/MonitorListener.java new file mode 100644 index 000000000..91ae0ac1a --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/monitor/MonitorListener.java @@ -0,0 +1,46 @@ +package com.alibaba.rocketmq.tools.monitor; + +import java.util.TreeMap; + +import com.alibaba.rocketmq.common.protocol.body.ConsumerRunningInfo; + + +/** + * 监控监听器 + */ +public interface MonitorListener { + /** + * 开始一轮监控 + */ + public void beginRound(); + + + /** + * 汇报消息堆积情况 + */ + public void reportUndoneMsgs(UndoneMsgs undoneMsgs); + + + /** + * 汇报消费失败情况 + */ + public void reportFailedMsgs(FailedMsgs failedMsgs); + + + /** + * 汇报消息删除情况 + */ + public void reportDeleteMsgsEvent(DeleteMsgsEvent deleteMsgsEvent); + + + /** + * 汇报Consumer内部运行数据结构 + */ + public void reportConsumerRunningInfo(TreeMap criTable); + + + /** + * 结束一轮监控 + */ + public void endRound(); +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/monitor/MonitorService.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/monitor/MonitorService.java new file mode 100644 index 000000000..1eee9afe4 --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/monitor/MonitorService.java @@ -0,0 +1,332 @@ +package com.alibaba.rocketmq.tools.monitor; + +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map.Entry; +import java.util.Random; +import java.util.TreeMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import org.slf4j.Logger; + +import com.alibaba.rocketmq.client.consumer.DefaultMQPullConsumer; +import com.alibaba.rocketmq.client.consumer.DefaultMQPushConsumer; +import com.alibaba.rocketmq.client.consumer.PullResult; +import com.alibaba.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext; +import com.alibaba.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus; +import com.alibaba.rocketmq.client.consumer.listener.MessageListenerConcurrently; +import com.alibaba.rocketmq.client.exception.MQBrokerException; +import com.alibaba.rocketmq.client.exception.MQClientException; +import com.alibaba.rocketmq.client.log.ClientLogger; +import com.alibaba.rocketmq.common.MQVersion; +import com.alibaba.rocketmq.common.MixAll; +import com.alibaba.rocketmq.common.ThreadFactoryImpl; +import com.alibaba.rocketmq.common.admin.ConsumeStats; +import com.alibaba.rocketmq.common.admin.OffsetWrapper; +import com.alibaba.rocketmq.common.message.MessageExt; +import com.alibaba.rocketmq.common.message.MessageQueue; +import com.alibaba.rocketmq.common.protocol.body.Connection; +import com.alibaba.rocketmq.common.protocol.body.ConsumerConnection; +import com.alibaba.rocketmq.common.protocol.body.ConsumerRunningInfo; +import com.alibaba.rocketmq.common.protocol.body.TopicList; +import com.alibaba.rocketmq.common.protocol.topic.OffsetMovedEvent; +import com.alibaba.rocketmq.remoting.RPCHook; +import com.alibaba.rocketmq.remoting.exception.RemotingException; +import com.alibaba.rocketmq.tools.admin.DefaultMQAdminExt; + + +public class MonitorService { + private final Logger log = ClientLogger.getLog(); + private final ScheduledExecutorService scheduledExecutorService = Executors + .newSingleThreadScheduledExecutor(new ThreadFactoryImpl("MonitorService")); + + private final MonitorConfig monitorConfig; + + private final MonitorListener monitorListener; + + private final DefaultMQAdminExt defaultMQAdminExt; + private final DefaultMQPullConsumer defaultMQPullConsumer = new DefaultMQPullConsumer( + MixAll.TOOLS_CONSUMER_GROUP); + private final DefaultMQPushConsumer defaultMQPushConsumer = new DefaultMQPushConsumer( + MixAll.MONITOR_CONSUMER_GROUP); + + + public MonitorService(MonitorConfig monitorConfig, MonitorListener monitorListener, RPCHook rpcHook) { + this.monitorConfig = monitorConfig; + this.monitorListener = monitorListener; + + this.defaultMQAdminExt = new DefaultMQAdminExt(rpcHook); + this.defaultMQAdminExt.setInstanceName(instanceName()); + this.defaultMQAdminExt.setNamesrvAddr(monitorConfig.getNamesrvAddr()); + + this.defaultMQPullConsumer.setInstanceName(instanceName()); + this.defaultMQPullConsumer.setNamesrvAddr(monitorConfig.getNamesrvAddr()); + + this.defaultMQPushConsumer.setInstanceName(instanceName()); + this.defaultMQPushConsumer.setNamesrvAddr(monitorConfig.getNamesrvAddr()); + try { + this.defaultMQPushConsumer.setConsumeThreadMin(1); + this.defaultMQPushConsumer.setConsumeThreadMax(1); + this.defaultMQPushConsumer.subscribe(MixAll.OFFSET_MOVED_EVENT, "*"); + this.defaultMQPushConsumer.registerMessageListener(new MessageListenerConcurrently() { + + @Override + public ConsumeConcurrentlyStatus consumeMessage(List msgs, + ConsumeConcurrentlyContext context) { + try { + OffsetMovedEvent ome = + OffsetMovedEvent.decode(msgs.get(0).getBody(), OffsetMovedEvent.class); + + DeleteMsgsEvent deleteMsgsEvent = new DeleteMsgsEvent(); + deleteMsgsEvent.setOffsetMovedEvent(ome); + deleteMsgsEvent.setEventTimestamp(msgs.get(0).getStoreTimestamp()); + + MonitorService.this.monitorListener.reportDeleteMsgsEvent(deleteMsgsEvent); + } + catch (Exception e) { + } + + return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; + } + }); + } + catch (MQClientException e) { + } + } + + + private String instanceName() { + String name = + System.currentTimeMillis() + new Random().nextInt() + this.monitorConfig.getNamesrvAddr(); + + return "MonitorService_" + name.hashCode(); + } + + + private void startScheduleTask() { + this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + try { + MonitorService.this.doMonitorWork(); + } + catch (Exception e) { + log.error("doMonitorWork Exception", e); + } + } + }, 1000 * 20, this.monitorConfig.getRoundInterval(), TimeUnit.MILLISECONDS); + } + + + public void start() throws MQClientException { + this.defaultMQPullConsumer.start(); + this.defaultMQAdminExt.start(); + this.defaultMQPushConsumer.start(); + this.startScheduleTask(); + } + + + public void shutdown() { + this.defaultMQPullConsumer.shutdown(); + this.defaultMQAdminExt.shutdown(); + this.defaultMQPushConsumer.shutdown(); + } + + + public void doMonitorWork() throws RemotingException, MQClientException, InterruptedException { + long beginTime = System.currentTimeMillis(); + this.monitorListener.beginRound(); + + TopicList topicList = defaultMQAdminExt.fetchAllTopicList(); + for (String topic : topicList.getTopicList()) { + if (topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) { + String consumerGroup = topic.substring(MixAll.RETRY_GROUP_TOPIC_PREFIX.length()); + // 监控消费进度 + try { + this.reportUndoneMsgs(consumerGroup); + } + catch (Exception e) { + // log.error("reportUndoneMsgs Exception", e); + } + + // 监控每个Consumer内存状态 + try { + this.reportConsumerRunningInfo(consumerGroup); + } + catch (Exception e) { + // log.error("reportConsumerRunningInfo Exception", e); + } + } + } + this.monitorListener.endRound(); + long spentTimeMills = System.currentTimeMillis() - beginTime; + log.info("Execute one round monitor work, spent timemills: {}", spentTimeMills); + } + + + public void reportConsumerRunningInfo(final String consumerGroup) throws InterruptedException, + MQBrokerException, RemotingException, MQClientException { + ConsumerConnection cc = defaultMQAdminExt.examineConsumerConnectionInfo(consumerGroup); + TreeMap infoMap = new TreeMap(); + for (Connection c : cc.getConnectionSet()) { + String clientId = c.getClientId(); + // 低于3.1.8版本，不支持此功能 + if (c.getVersion() < MQVersion.Version.V3_1_8_SNAPSHOT.ordinal()) { + continue; + } + + try { + ConsumerRunningInfo info = + defaultMQAdminExt.getConsumerRunningInfo(consumerGroup, clientId, false); + infoMap.put(clientId, info); + } + catch (Exception e) { + } + } + + if (!infoMap.isEmpty()) { + this.monitorListener.reportConsumerRunningInfo(infoMap); + } + } + + + private void reportFailedMsgs(final String consumerGroup, final String topic) { + + } + + + private void reportUndoneMsgs(final String consumerGroup) { + ConsumeStats cs = null; + try { + cs = defaultMQAdminExt.examineConsumeStats(consumerGroup); + } + catch (Exception e) { + return; + } + + ConsumerConnection cc = null; + try { + cc = defaultMQAdminExt.examineConsumerConnectionInfo(consumerGroup); + } + catch (Exception e) { + return; + } + + if (cs != null) { + // 按照Topic拆分 + HashMap csByTopic = new HashMap(); + { + Iterator> it = cs.getOffsetTable().entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + MessageQueue mq = next.getKey(); + OffsetWrapper ow = next.getValue(); + ConsumeStats csTmp = csByTopic.get(mq.getTopic()); + if (null == csTmp) { + csTmp = new ConsumeStats(); + csByTopic.put(mq.getTopic(), csTmp); + } + + csTmp.getOffsetTable().put(mq, ow); + } + } + + // 按照Topic开始报警 + { + Iterator> it = csByTopic.entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + UndoneMsgs undoneMsgs = new UndoneMsgs(); + undoneMsgs.setConsumerGroup(consumerGroup); + undoneMsgs.setTopic(next.getKey()); + this.computeUndoneMsgs(undoneMsgs, next.getValue()); + this.monitorListener.reportUndoneMsgs(undoneMsgs); + this.reportFailedMsgs(consumerGroup, next.getKey()); + } + } + } + } + + + private void computeUndoneMsgs(final UndoneMsgs undoneMsgs, final ConsumeStats consumeStats) { + long total = 0; + long singleMax = 0; + long delayMax = 0; + Iterator> it = consumeStats.getOffsetTable().entrySet().iterator(); + while (it.hasNext()) { + Entry next = it.next(); + MessageQueue mq = next.getKey(); + OffsetWrapper ow = next.getValue(); + long diff = ow.getBrokerOffset() - ow.getConsumerOffset(); + + if (diff > singleMax) { + singleMax = diff; + } + + if (diff > 0) { + total += diff; + } + + // Delay + if (ow.getLastTimestamp() > 0) { + try { + long maxOffset = this.defaultMQPullConsumer.maxOffset(mq); + if (maxOffset > 0) { + PullResult pull = this.defaultMQPullConsumer.pull(mq, "*", maxOffset - 1, 1); + switch (pull.getPullStatus()) { + case FOUND: + long delay = + pull.getMsgFoundList().get(0).getStoreTimestamp() - ow.getLastTimestamp(); + if (delay > delayMax) { + delayMax = delay; + } + break; + case NO_MATCHED_MSG: + case NO_NEW_MSG: + case OFFSET_ILLEGAL: + break; + default: + break; + } + } + } + catch (Exception e) { + } + } + } + + undoneMsgs.setUndoneMsgsTotal(total); + undoneMsgs.setUndoneMsgsSingleMQ(singleMax); + undoneMsgs.setUndoneMsgsDelayTimeMills(delayMax); + } + + + public static void main(String[] args) throws MQClientException { + main0(args, null); + } + + + public static void main0(String[] args, RPCHook rpcHook) throws MQClientException { + final MonitorService monitorService = + new MonitorService(new MonitorConfig(), new DefaultMonitorListener(), rpcHook); + monitorService.start(); + + Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() { + private volatile boolean hasShutdown = false; + + + @Override + public void run() { + synchronized (this) { + if (!this.hasShutdown) { + this.hasShutdown = true; + monitorService.shutdown(); + } + } + } + }, "ShutdownHook")); + } +} diff --git a/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/monitor/UndoneMsgs.java b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/monitor/UndoneMsgs.java new file mode 100644 index 000000000..6fe0e9313 --- /dev/null +++ b/rocketmq-tools/src/main/java/com/alibaba/rocketmq/tools/monitor/UndoneMsgs.java @@ -0,0 +1,70 @@ +package com.alibaba.rocketmq.tools.monitor; + +public class UndoneMsgs { + private String consumerGroup; + private String topic; + // 堆积的消息总数 + private long undoneMsgsTotal; + // 单个队列堆积的最多消息数 + private long undoneMsgsSingleMQ; + // Delay的最长时间，单位秒 + private long undoneMsgsDelayTimeMills; + + + public String getConsumerGroup() { + return consumerGroup; + } + + + public void setConsumerGroup(String consumerGroup) { + this.consumerGroup = consumerGroup; + } + + + public String getTopic() { + return topic; + } + + + public void setTopic(String topic) { + this.topic = topic; + } + + + public long getUndoneMsgsTotal() { + return undoneMsgsTotal; + } + + + public void setUndoneMsgsTotal(long undoneMsgsTotal) { + this.undoneMsgsTotal = undoneMsgsTotal; + } + + + public long getUndoneMsgsSingleMQ() { + return undoneMsgsSingleMQ; + } + + + public void setUndoneMsgsSingleMQ(long undoneMsgsSingleMQ) { + this.undoneMsgsSingleMQ = undoneMsgsSingleMQ; + } + + + public long getUndoneMsgsDelayTimeMills() { + return undoneMsgsDelayTimeMills; + } + + + public void setUndoneMsgsDelayTimeMills(long undoneMsgsDelayTimeMills) { + this.undoneMsgsDelayTimeMills = undoneMsgsDelayTimeMills; + } + + + @Override + public String toString() { + return "UndoneMsgs [consumerGroup=" + consumerGroup + ", topic=" + topic + ", undoneMsgsTotal=" + + undoneMsgsTotal + ", undoneMsgsSingleMQ=" + undoneMsgsSingleMQ + + ", undoneMsgsDelayTimeMills=" + undoneMsgsDelayTimeMills + "]"; + } +} diff --git a/test/consumer.sh b/test/consumer.sh new file mode 100644 index 000000000..fbeb1789f --- /dev/null +++ b/test/consumer.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +# +# $Id: consumer.sh 1831 2013-05-16 01:39:51Z shijia.wxr $ +# +sh ./runclass.sh com.alibaba.rocketmq.example.operation.Consumer $@ diff --git a/test/producer.sh b/test/producer.sh new file mode 100644 index 000000000..058c6036c --- /dev/null +++ b/test/producer.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +# +# $Id: producer.sh 1831 2013-05-16 01:39:51Z shijia.wxr $ +# +sh ./runclass.sh com.alibaba.rocketmq.example.operation.Producer $@ diff --git a/test/runclass.sh b/test/runclass.sh new file mode 100644 index 000000000..5b6380310 --- /dev/null +++ b/test/runclass.sh @@ -0,0 +1,31 @@ +#!/bin/sh + +# +# $Id: runclass.sh 857 2012-12-24 06:31:31Z shijia.wxr $ +# + +if [ $# -lt 1 ]; +then + echo "USAGE: $0 classname opts" + exit 1 +fi + +BASE_DIR=$(dirname $0)/.. +CLASSPATH=.:${BASE_DIR}/conf:${CLASSPATH} + +JAVA_OPT_1="-server -Xms1g -Xmx1g -Xmn256m -XX:PermSize=128m -XX:MaxPermSize=320m" +JAVA_OPT_2="-XX:+UseConcMarkSweepGC -XX:+UseCMSCompactAtFullCollection -XX:CMSInitiatingOccupancyFraction=70 -XX:+CMSParallelRemarkEnabled -XX:SoftRefLRUPolicyMSPerMB=0 -XX:+CMSClassUnloadingEnabled -XX:SurvivorRatio=8 -XX:+DisableExplicitGC" +JAVA_OPT_3="-verbose:gc -Xloggc:${HOME}/rocketmq_client_gc.log -XX:+PrintGCDetails -XX:+PrintGCDateStamps" +JAVA_OPT_4="-XX:-OmitStackTraceInFastThrow" +JAVA_OPT_5="-Djava.ext.dirs=${BASE_DIR}/lib" +JAVA_OPT_6="-cp ${CLASSPATH}" + +if [ -z "$JAVA_HOME" ]; then + JAVA_HOME=/opt/taobao/java +fi + +JAVA="$JAVA_HOME/bin/java" + +JAVA_OPTS="${JAVA_OPT_1} ${JAVA_OPT_2} ${JAVA_OPT_3} ${JAVA_OPT_4} ${JAVA_OPT_5} ${JAVA_OPT_6}" + +$JAVA $JAVA_OPTS $@