Add statistics for CRUD operations on router

.github/workflows/test_on_push.yaml

@@ -13,13 +13,21 @@ jobs:
       matrix:
         # We need 1.10.6 here to check that module works with
         # old Tarantool versions that don't have "tuple-keydef"/"tuple-merger" support.
-        tarantool-version: ["1.10.6", "1.10", "2.2", "2.3", "2.4", "2.5", "2.6", "2.7"]
+        tarantool-version: ["1.10.6", "1.10", "2.2", "2.3", "2.4", "2.5", "2.6", "2.7", "2.8"]
+        metrics-version: [""]
         remove-merger: [false]
         include:
+          - tarantool-version: "1.10"
+            metrics-version: "0.12.0"
           - tarantool-version: "2.7"
             remove-merger: true
+          - tarantool-version: "2.8"
+            metrics-version: "0.1.8"
+          - tarantool-version: "2.8"
+            metrics-version: "0.10.0"
           - tarantool-version: "2.8"
             coveralls: true
+            metrics-version: "0.12.0"
       fail-fast: false
     runs-on: [ubuntu-latest]
     steps:
@@ -47,6 +55,10 @@ jobs:
           tarantool --version
           ./deps.sh
 
+      - name: Install metrics
+        if: matrix.metrics-version != ''
+        run: tarantoolctl rocks install metrics ${{ matrix.metrics-version }}
+
       - name: Remove external merger if needed
         if: ${{ matrix.remove-merger }}
         run: rm .rocks/lib/tarantool/tuple/merger.so
@@ -66,11 +78,47 @@ jobs:
         run: make -C build coveralls
         if: ${{ matrix.coveralls }}
 
+  run-perf-tests-ce:
+    if: |
+      github.event_name == 'push' ||
+      github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository
+    strategy:
+      matrix:
+        tarantool-version: ["1.10", "2.8"]
+        metrics-version: ["0.12.0"]
+      fail-fast: false
+    runs-on: [ubuntu-latest]
+    steps:
+      - uses: actions/checkout@master
+
+      - name: Setup Tarantool CE 
+        uses: tarantool/setup-tarantool@v1
+        with:
+          tarantool-version: ${{ matrix.tarantool-version }}
+
+      - name: Install requirements for community
+        run: |
+          tarantool --version
+          ./deps.sh
+
+      - name: Install metrics
+        run: tarantoolctl rocks install metrics ${{ matrix.metrics-version }}
+
+      # This server starts and listen on 8084 port that is used for tests
+      - name: Stop Mono server
+        run: sudo kill -9 $(sudo lsof -t -i tcp:8084) || true
+
+      - run: cmake -S . -B build
+
+      - name: Run performance tests
+        run: make -C build performance
+
   run-tests-ee:
     if: github.event_name == 'push'
     strategy:
       matrix:
         bundle_version: [ "1.10.11-0-gf0b0e7ecf-r422", "2.7.3-0-gdddf926c3-r422" ]
+        metrics-version: ["", "0.12.0"]
       fail-fast: false
     runs-on: [ ubuntu-latest ]
     steps:
@@ -86,6 +134,12 @@ jobs:
           tarantool --version
           ./deps.sh
 
+      - name: Install metrics
+        if: matrix.metrics-version != ''
+        run: |
+          source tarantool-enterprise/env.sh
+          tarantoolctl rocks install metrics ${{ matrix.metrics-version }}
+
       # This server starts and listen on 8084 port that is used for tests
       - name: Stop Mono server
         run: sudo kill -9 $(sudo lsof -t -i tcp:8084) || true

.gitignore

@@ -8,3 +8,5 @@ CMakeCache.txt
 CMakeFiles/
 luacov.report.out
 luacov.stats.out
+build/*.cmake
+build/Makefile

.luacheckrc

@@ -3,3 +3,4 @@ globals = {'box', 'utf8', 'checkers', '_TARANTOOL'}
 include_files = {'**/*.lua', '*.luacheckrc', '*.rockspec'}
 exclude_files = {'**/*.rocks/', 'tmp/', 'tarantool-enterprise/'}
 max_line_length = 120
+max_comment_line_length = 150

CHANGELOG.md

@@ -8,6 +8,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 ## [Unreleased]
 
 ### Added
+* Statistics for CRUD operations on router (#224).
+* Integrate CRUD statistics with [`metrics`](https://github.com/tarantool/metrics) (#224).
 
 ### Changed
 

CMakeLists.txt

@@ -36,6 +36,14 @@ add_custom_target(luatest
   COMMENT "Run regression tests"
 )
 
+set(PERFORMANCE_TESTS_SUBDIR "test/performance")
+
+add_custom_target(performance
+  COMMAND PERF_MODE_ON=true ${LUATEST} -v -c ${PERFORMANCE_TESTS_SUBDIR}
+  WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+  COMMENT "Run performance tests"
+)
+
 add_custom_target(coverage
   COMMAND ${LUACOV} ${PROJECT_SOURCE_DIR} && grep -A999 '^Summary' ${CODE_COVERAGE_REPORT}
   DEPENDS ${CODE_COVERAGE_STATS}

README.md

@@ -674,6 +674,171 @@ Combinations of `mode`, `prefer_replica` and `balance` options lead to:
   * prefer_replica, balance -
     [vshard call `callbre`](https://www.tarantool.io/en/doc/latest/reference/reference_rock/vshard/vshard_api/#router-api-callbre)
 
+### Statistics
+
+`crud` routers can provide statistics on called operations.
+```lua
+-- Enable statistics collect.
+crud.cfg{ stats = true }
+
+-- Returns table with statistics information.
+crud.stats()
+
+-- Returns table with statistics information for specific space.
+crud.stats('my_space')
+
+-- Disables statistics collect and destroys all collectors.
+crud.cfg{ stats = false }
+
+-- Destroys all statistics collectors and creates them again.
+crud.reset_stats()
+```
+
+If [`metrics`](https://github.com/tarantool/metrics) `0.10.0` or greater
+found, metrics collectors will be used by default to store statistics
+instead of local collectors. Quantiles in metrics summary collections
+are disabled by default. You can manually choose driver and enable quantiles.
+```lua
+-- Use simple local collectors (default if no required metrics version found).
+crud.cfg{ stats = true, stats_driver = 'local' }
+
+-- Use metrics collectors (default if metrics rock found).
+crud.cfg{ stats = true, stats_driver = 'metrics' }
+
+-- Use metrics collectors with 0.99 quantiles.
+crud.cfg{ stats = true, stats_driver = 'metrics', stats_quantiles = true }
+```
+
+You can use `crud.cfg` to check current stats state.
+```lua
+crud.cfg
+---
+- stats_quantiles: true
+  stats: true
+  stats_driver: metrics
+...
+```
+Performance overhead is 3-10% in case of `local` driver and
+5-15% in case of `metrics` driver, up to 20% for `metrics` with quantiles.
+
+Beware that iterating through `crud.cfg` with pairs is not supported yet,
+refer to [tarantool/crud#265](https://github.com/tarantool/crud/issues/265).
+
+Format is as follows.
+```lua
+crud.stats()
+---
+- spaces:
+    my_space:
+      insert:
+        ok:
+          latency: 0.002
+          count: 19800
+          time: 39.6
+        error:
+          latency: 0.000001
+          count: 4
+          time: 0.000004
+...
+crud.stats('my_space')
+---
+- insert:
+    ok:
+      latency: 0.002
+      count: 19800
+      time: 39.6
+    error:
+      latency: 0.000001
+      count: 4
+      time: 0.000004
+...
+```
+`spaces` section contains statistics for each observed space.
+If operation has never been called for a space, the corresponding
+field will be empty. If no requests has been called for a
+space, it will not be represented. Space data is based on
+client requests rather than storages schema, so requests
+for non-existing spaces are also collected.
+
+Possible statistics operation labels are
+`insert` (for `insert` and `insert_object` calls),
+`get`, `replace` (for `replace` and `replace_object` calls), `update`,
+`upsert` (for `upsert` and `upsert_object` calls), `delete`,
+`select` (for `select` and `pairs` calls), `truncate`, `len`, `count`
+and `borders` (for `min` and `max` calls).
+
+Each operation section consists of different collectors
+for success calls and error (both error throw and `nil, err`)
+returns. `count` is the total requests count since instance start
+or stats restart. `latency` is the 0.99 quantile of request execution
+time if `metrics` driver used and quantiles enabled,
+otherwise `latency` is the total average.
+`time` is the total time of requests execution.
+
+In [`metrics`](https://www.tarantool.io/en/doc/latest/book/monitoring/)
+registry statistics are stored as `tnt_crud_stats` metrics
+with `operation`, `status` and `name` labels.
+```
+metrics:collect()
+---
+- - label_pairs:
+      status: ok
+      operation: insert
+      name: customers
+    value: 221411
+    metric_name: tnt_crud_stats_count
+  - label_pairs:
+      status: ok
+      operation: insert
+      name: customers
+    value: 10.49834896344692
+    metric_name: tnt_crud_stats_sum
+  - label_pairs:
+      status: ok
+      operation: insert
+      name: customers
+      quantile: 0.99
+    value: 0.00023606420935973
+    metric_name: tnt_crud_stats
+...
+```
+
+`select` section additionally contains `details` collectors.
+```lua
+crud.stats('my_space').select.details
+---
+- map_reduces: 4
+  tuples_fetched: 10500
+  tuples_lookup: 238000
+...
+```
+`map_reduces` is the count of planned map reduces (including those not
+executed successfully). `tuples_fetched` is the count of tuples fetched
+from storages during execution, `tuples_lookup` is the count of tuples
+looked up on storages while collecting responses for calls (including
+scrolls for multibatch requests). Details data is updated as part of
+the request process, so you may get new details before `select`/`pairs`
+call is finished and observed with count, latency and time collectors.
+In [`metrics`](https://www.tarantool.io/en/doc/latest/book/monitoring/)
+registry they are stored as `tnt_crud_map_reduces`,
+`tnt_crud_tuples_fetched` and `tnt_crud_tuples_lookup` metrics
+with `{ operation = 'select', name = space_name }` labels.
+
+Since `pairs` request behavior differs from any other crud request, its
+statistics collection also has specific behavior. Statistics (`select`
+section) are updated after `pairs` cycle is finished: you
+either have iterated through all records or an error was thrown.
+If your pairs cycle was interrupted with `break`, statistics will
+be collected when pairs objects are cleaned up with Lua garbage
+collector.
+
+Statistics are preserved between package reloads. Statistics are preserved
+between [Tarantool Cartridge role reloads](https://www.tarantool.io/en/doc/latest/book/cartridge/cartridge_api/modules/cartridge.roles/#reload)
+if you use CRUD Cartridge roles. Beware that metrics 0.12.0 and below do not
+support preserving stats between role reload
+(see [tarantool/metrics#334](https://github.com/tarantool/metrics/issues/334)),
+thus this feature will be unsupported for `metrics` driver.
+
 ## Cartridge roles
 
 `cartridge.roles.crud-storage` is a Tarantool Cartridge role that depends on the

cartridge/roles/crud-router.lua

@@ -1,8 +1,10 @@
 local crud = require('crud')
+local stash = require('crud.common.stash')
 
 -- removes routes that changed in config and adds new routes
 local function init()
     crud.init_router()
+    stash.setup_cartridge_reload()
 end
 
 local function stop()

cartridge/roles/crud-storage.lua

@@ -1,7 +1,9 @@
 local crud = require('crud')
+local stash = require('crud.common.stash')
 
 local function init()
     crud.init_storage()
+    stash.setup_cartridge_reload()
 end
 
 local function stop()