sv6/include/lb.hh at master · aclements/sv6 · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
#pragma once
#define NCPU_PER_SOCKET (NCPU/NSOCKET)

#include "rnd.hh"
#include "percpu.hh"

template<int N>
struct random_permutation {
 public:
  random_permutation() : i_(0) {
    assert(N <= 256);
    for (int i = 0; i < N; i++)
      x_[i] = i;
  }

  void reset() {
    i_ = 0;
  }

  int next() {
#if CODEX
    int r = rnd() % (N - i_);
#else
    int r = rdtsc() % (N - i_);
#endif
    std::swap(x_[i_], x_[r]);
    return x_[i_++];
  }

 private:
  char x_[N];
  int i_;
};

// For wrapping into <percpu>, which maybe important to avoid false sharing
struct persocket {
  random_permutation<NCPU_PER_SOCKET-1> perm;
};

struct othersocket {
  random_permutation<NCPU-NCPU_PER_SOCKET> perm;
};

template<class Pool>
class balance_pool {
 public:
  balance_pool(u64 max) : balance_max_(max) {}

  bool balanced() const {
    Pool* thispool = (Pool*) this;
    u64 c = thispool->balance_count();
    return c != 0 && c != balance_max_;
  }

  void balance_with(Pool* otherpool) {
    Pool* thispool = (Pool*) this;
    u64 thisbal = thispool->balance_count();
    u64 otherbal = otherpool->balance_count();

    if (thisbal < otherbal) {
      otherpool->balance_move_to(thispool);
    } else if (otherbal > thisbal) {
      thispool->balance_move_to(otherpool);
    }
  }

 private:
  u64 balance_max_;
};

template<class PoolDir, class Pool>
class balancer {
 public:
  balancer(const PoolDir* bd) : bd_(bd) {}
  ~balancer() {}

  void balance() {
    int myid = mycpu()->id;
    Pool* thispool = bd_->balance_get(myid);
    if (!thispool)
      return;

    u64 sock_first_core = (myid / NCPU_PER_SOCKET) * NCPU_PER_SOCKET;
    u64 sock_myoff = myid % NCPU_PER_SOCKET;

    rpsock_->perm.reset();

    for (int i = 0; i < NCPU_PER_SOCKET-1; i++) {
      int bal_id = sock_first_core +
        ((sock_myoff + 1 + rpsock_->perm.next()) % NCPU_PER_SOCKET);
      Pool* otherpool = bd_->balance_get(bal_id);
      if (otherpool && (thispool != otherpool)) {
        thispool->balance_with(otherpool);
        if (thispool->balanced())
          return;
      }
    }

    rpother_->perm.reset();
    for (int i = 0; i < NCPU-NCPU_PER_SOCKET; i++) {
      int bal_id = (sock_first_core + NCPU_PER_SOCKET +
                 rpother_->perm.next()) % NCPU;
      Pool* otherpool = bd_->balance_get(bal_id);
      if (otherpool && (thispool != otherpool)) {
        thispool->balance_with(otherpool);
        if (thispool->balanced())
          break;
      }
    }
  }

 private:
  const PoolDir* const bd_;
  percpu<persocket,NO_CRITICAL> rpsock_;
  percpu<othersocket,NO_CRITICAL> rpother_;
};