Skip to content

Commit 2f61c62

Browse files
BewareMyPowerBewareMyPower
authored
Speed up the process to start standalone for tests (#236)
### Motivation It took too much time to start Pulsar standalone for tests because of the slow JVM startup when running `pulsar-admin` commands. ### Modifications Use `curl` to send REST requests directly. After this change, the startup time reduced from 1m39.597s to 32.451s in my local env.
1 parent 0088a64 commit 2f61c62

File tree

2 files changed

+119
-58
lines changed

2 files changed

+119
-58
lines changed

‎build-support/start-mim-test-service-inside-container.sh‎

Lines changed: 34 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,24 @@ bin/pulsar tokens create \
3131
--secret-key file:///pulsar/data/tokens/secret.key \
3232
> /pulsar/data/tokens/token.txt
3333

34+
# Unset the HTTP proxy to avoid the REST requests being affected
35+
export http_proxy=
36+
TOKEN=$(bin/pulsar tokens create \
37+
--subject superUser \
38+
--secret-key file:///pulsar/data/tokens/secret.key)
39+
40+
# Create "standalone" cluster if it does not exist
41+
put() {
42+
curl -H "Authorization: Bearer $TOKEN" \
43+
-L http://localhost:8081/admin/v2/$1 \
44+
-H 'Content-Type: application/json' \
45+
-X PUT \
46+
-d $(echo $2 | sed 's/ //g')
47+
}
48+
3449
export PULSAR_STANDALONE_CONF=test-conf/standalone-ssl-mim.conf
3550
export PULSAR_PID_DIR=/tmp
51+
sed -i 's/immediateFlush: false/immediateFlush: true/' conf/log4j2.yaml
3652
bin/pulsar-daemon start standalone \
3753
--no-functions-worker --no-stream-storage \
3854
--bookkeeper-dir data/bookkeeper
@@ -42,25 +58,28 @@ until curl http://localhost:8081/metrics > /dev/null 2>&1 ; do sleep 1; done
4258

4359
echo "-- Pulsar service is ready -- Configure permissions"
4460

45-
export PULSAR_CLIENT_CONF=test-conf/client-ssl-mim.conf
46-
4761
# Create "standalone" cluster if it does not exist
48-
bin/pulsar-admin clusters list | grep -q '^standalone$' ||
49-
bin/pulsar-admin clusters create \
50-
standalone \
51-
--url http://localhost:8081/ \
52-
--url-secure https://localhost:8444/ \
53-
--broker-url pulsar://localhost:6652/ \
54-
--broker-url-secure pulsar+ssl://localhost:6653/
62+
put clusters/standalone '{
63+
"serviceUrl": "http://localhost:8081/",
64+
"serviceUrlTls": "https://localhost:8444/",
65+
"brokerServiceUrl": "pulsar://localhost:6652/",
66+
"brokerServiceUrlTls": "pulsar+ssl://localhost:6653/"
67+
}'
5568

5669
# Create "private" tenant
57-
bin/pulsar-admin tenants create private -r "" -c "standalone"
70+
put tenants/private '{
71+
"adminRoles": [],
72+
"allowedClusters": ["standalone"]
73+
}'
5874

5975
# Create "private/auth" with required authentication
60-
bin/pulsar-admin namespaces create private/auth --clusters standalone
61-
62-
bin/pulsar-admin namespaces grant-permission private/auth \
63-
--actions produce,consume \
64-
--role "token-principal"
76+
put namespaces/private/auth '{
77+
"auth_policies": {
78+
"namespace_auth": {
79+
"token-principal": ["produce", "consume"]
80+
}
81+
},
82+
"replication_clusters": ["standalone"]
83+
}'
6584

6685
echo "-- Ready to start tests"

‎build-support/start-test-service-inside-container.sh‎

Lines changed: 85 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,26 @@ bin/pulsar tokens create \
3131
--secret-key file:///pulsar/data/tokens/secret.key \
3232
> /pulsar/data/tokens/token.txt
3333

34+
# Unset the HTTP proxy to avoid the REST requests being affected
35+
export http_proxy=
36+
TOKEN=$(bin/pulsar tokens create \
37+
--subject superUser \
38+
--secret-key file:///pulsar/data/tokens/secret.key)
39+
40+
# Create "standalone" cluster if it does not exist
41+
put() {
42+
curl -H "Authorization: Bearer $TOKEN" \
43+
-L http://localhost:8080/admin/v2/$1 \
44+
-H 'Content-Type: application/json' \
45+
-X PUT \
46+
-d $(echo $2 | sed 's/ //g')
47+
}
48+
3449
export PULSAR_STANDALONE_CONF=test-conf/standalone-ssl.conf
3550
export PULSAR_PID_DIR=/tmp
51+
export PULSAR_STANDALONE_USE_ZOOKEEPER=1
52+
sed -i 's/immediateFlush: false/immediateFlush: true/' conf/log4j2.yaml
53+
3654
bin/pulsar-daemon start standalone \
3755
--no-functions-worker --no-stream-storage \
3856
--bookkeeper-dir data/bookkeeper
@@ -42,60 +60,84 @@ until curl http://localhost:8080/metrics > /dev/null 2>&1 ; do sleep 1; done
4260

4361
echo "-- Pulsar service is ready -- Configure permissions"
4462

45-
export PULSAR_CLIENT_CONF=test-conf/client-ssl.conf
46-
47-
# Create "standalone" cluster if it does not exist
48-
bin/pulsar-admin clusters list | grep -q '^standalone$' ||
49-
bin/pulsar-admin clusters create \
50-
standalone \
51-
--url http://localhost:8080/ \
52-
--url-secure https://localhost:8443/ \
53-
--broker-url pulsar://localhost:6650/ \
54-
--broker-url-secure pulsar+ssl://localhost:6651/
55-
56-
# Update "public" tenant
57-
bin/pulsar-admin tenants create public -r "anonymous" -c "standalone"
58-
59-
# Update "public/default" with no auth required
60-
bin/pulsar-admin namespaces create public/default -c standalone
61-
bin/pulsar-admin namespaces grant-permission public/default \
62-
--actions produce,consume \
63-
--role "anonymous"
63+
put clusters/standalone '{
64+
"serviceUrl": "http://localhost:8080/",
65+
"serviceUrlTls": "https://localhost:8443/",
66+
"brokerServiceUrl": "pulsar://localhost:6650/",
67+
"brokerServiceUrlTls": "pulsar+ssl://localhost:6651/"
68+
}'
69+
70+
# Create "public" tenant
71+
put tenants/public '{
72+
"adminRoles": ["anonymous"],
73+
"allowedClusters": ["standalone"]
74+
}'
75+
76+
# Create "public/default" with no auth required
77+
put namespaces/public/default '{
78+
"auth_policies": {
79+
"namespace_auth": {
80+
"anonymous": ["produce", "consume"]
81+
}
82+
},
83+
"replication_clusters": ["standalone"]
84+
}'
6485

6586
# Create "public/default-2" with no auth required
66-
bin/pulsar-admin namespaces create public/default-2 \
67-
--clusters standalone
68-
bin/pulsar-admin namespaces grant-permission public/default-2 \
69-
--actions produce,consume \
70-
--role "anonymous"
87+
put namespaces/public/default-2 '{
88+
"auth_policies": {
89+
"namespace_auth": {
90+
"anonymous": ["produce", "consume"]
91+
}
92+
},
93+
"replication_clusters": ["standalone"]
94+
}'
7195

7296
# Create "public/default-3" with no auth required
73-
bin/pulsar-admin namespaces create public/default-3 \
74-
--clusters standalone
75-
bin/pulsar-admin namespaces grant-permission public/default-3 \
76-
--actions produce,consume \
77-
--role "anonymous"
97+
put namespaces/public/default-3 '{
98+
"auth_policies": {
99+
"namespace_auth": {
100+
"anonymous": ["produce", "consume"]
101+
}
102+
},
103+
"replication_clusters": ["standalone"]
104+
}'
78105

79106
# Create "public/default-4" with encryption required
80-
bin/pulsar-admin namespaces create public/default-4 \
81-
--clusters standalone
82-
bin/pulsar-admin namespaces grant-permission public/default-4 \
83-
--actions produce,consume \
84-
--role "anonymous"
85-
bin/pulsar-admin namespaces set-encryption-required public/default-4 -e
107+
put namespaces/public/default-4 '{
108+
"auth_policies": {
109+
"namespace_auth": {
110+
"anonymous": ["produce", "consume"]
111+
}
112+
},
113+
"encryption_required": true,
114+
"replication_clusters": ["standalone"]
115+
}'
86116

87117
# Create "public/test-backlog-quotas" to test backlog quotas policy
88-
bin/pulsar-admin namespaces create public/test-backlog-quotas \
89-
--clusters standalone
118+
put namespaces/public/test-backlog-quotas '{
119+
"auth_policies": {
120+
"namespace_auth": {
121+
"anonymous": ["produce", "consume"]
122+
}
123+
},
124+
"replication_clusters": ["standalone"]
125+
}'
90126

91127
# Create "private" tenant
92-
bin/pulsar-admin tenants create private -r "" -c "standalone"
128+
put tenants/private '{
129+
"adminRoles": [],
130+
"allowedClusters": ["standalone"]
131+
}'
93132

94133
# Create "private/auth" with required authentication
95-
bin/pulsar-admin namespaces create private/auth --clusters standalone
96-
97-
bin/pulsar-admin namespaces grant-permission private/auth \
98-
--actions produce,consume \
99-
--role "token-principal"
134+
put namespaces/private/auth '{
135+
"auth_policies": {
136+
"namespace_auth": {
137+
"token-principal": ["produce", "consume"]
138+
}
139+
},
140+
"replication_clusters": ["standalone"]
141+
}'
100142

101143
echo "-- Ready to start tests"

0 commit comments

Comments
 (0)