Skip to content

Commit

Permalink
Merge branch 'master' into pr/fetch-v10
Browse files Browse the repository at this point in the history
  • Loading branch information
@Nevon
Nevon authored Jul 20, 2020
2 parents b9effb0 + 46d5f54 commit b688f0e
Show file tree
Hide file tree
Showing 67 changed files with 980 additions and 552 deletions.
35 changes: 32 additions & 3 deletions azure-pipelines.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,9 @@ pr:
variables:
- group: website_secrets
- name: KAFKA_VERSION
value: 2.3
value: 2.4
- name: COMPOSE_FILE
value: docker-compose.2_3.yml
value: docker-compose.2_4.yml

####### Linter
jobs:
Expand Down Expand Up @@ -177,7 +177,36 @@ jobs:
condition: succeededOrFailed()
inputs:
testRunner: JUnit
testResultsFiles: '**/test-report.xml'
testResultsFiles: "**/test-report.xml"

- job: test_oauthbearer
displayName: 'OauthBearer'
condition: contains(variables['Build.SourceBranch'], 'refs/heads/master')
dependsOn: lint
variables:
- name: test_retries
value: 2
- name: COMPOSE_FILE
value: docker-compose.2_4_oauthbearer.yml
pool:
vmImage: 'Ubuntu 16.04'
steps:
- task: NodeTool@0
inputs:
versionSpec: '10.x'
- bash: ./scripts/pipeline/shouldRunTests.sh && SKIP_TESTS=true && echo "Only non-code has changed!" || test true
displayName: should skip tests
- bash: test $SKIP_TESTS && echo "Skipped!" || yarn install
displayName: yarn install
- bash: test $SKIP_TESTS && echo "Skipped!" || docker-compose -f ${COMPOSE_FILE} pull
displayName: docker-compose pull
- bash: test $SKIP_TESTS && echo "Skipped!" || yarn test:group:oauthbearer:ci
displayName: test
- task: PublishTestResults@2
condition: succeededOrFailed()
inputs:
testRunner: JUnit
testResultsFiles: "**/test-report.xml"

####### Deploy
- job: npm_release
Expand Down
143 changes: 143 additions & 0 deletions docker-compose.2_4.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,143 @@
version: '2'
services:
zookeeper:
image: confluentinc/cp-zookeeper:latest
hostname: zookeeper
container_name: zookeeper
ports:
- '2181:2181'
environment:
ZOOKEEPER_CLIENT_PORT: '2181'
ZOOKEEPER_TICK_TIME: '2000'
KAFKA_OPTS: '-Djava.security.auth.login.config=/etc/kafka/server-jaas.conf -Dzookeeper.authProvider.1=org.apache.zookeeper.server.auth.SASLAuthenticationProvider'
volumes:
- ./testHelpers/kafka/server-jaas.conf:/etc/kafka/server-jaas.conf:ro,z

kafka1:
image: confluentinc/cp-kafka:5.4.2
hostname: kafka1
container_name: kafka1
labels:
- 'custom.project=kafkajs'
- 'custom.service=kafka1'
depends_on:
- zookeeper
ports:
- '29092:29092'
- '9092:9092'
- '29093:29093'
- '9093:9093'
- '29094:29094'
- '9094:9094'
environment:
KAFKA_BROKER_ID: '0'
KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT,SSL:SSL,SSL_HOST:SSL,SASL_SSL:SASL_SSL,SASL_SSL_HOST:SASL_SSL
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka1:29092,PLAINTEXT_HOST://localhost:9092,SSL://kafka1:29093,SSL_HOST://localhost:9093,SASL_SSL://kafka1:29094,SASL_SSL_HOST://localhost:9094
KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
KAFKA_DELETE_TOPIC_ENABLE: 'true'
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: '0'
KAFKA_SSL_KEYSTORE_FILENAME: 'kafka.server.keystore.jks'
KAFKA_SSL_KEYSTORE_CREDENTIALS: 'keystore_creds'
KAFKA_SSL_KEY_CREDENTIALS: 'sslkey_creds'
KAFKA_SSL_TRUSTSTORE_FILENAME: 'kafka.server.truststore.jks'
KAFKA_SSL_TRUSTSTORE_CREDENTIALS: 'truststore_creds'
KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL: 'PLAIN'
KAFKA_SASL_ENABLED_MECHANISMS: 'PLAIN,SCRAM-SHA-256,SCRAM-SHA-512'
KAFKA_OPTS: '-Djava.security.auth.login.config=/opt/kafka/config/server-jaas.conf'
# suppress verbosity
# https://github.com/confluentinc/cp-docker-images/blob/master/debian/kafka/include/etc/confluent/docker/log4j.properties.template
KAFKA_LOG4J_LOGGERS: 'kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO'
volumes:
- ./testHelpers/certs/kafka.server.keystore.jks:/etc/kafka/secrets/kafka.server.keystore.jks:ro,z
- ./testHelpers/certs/kafka.server.truststore.jks:/etc/kafka/secrets/kafka.server.truststore.jks:ro,z
- ./testHelpers/certs/keystore_creds:/etc/kafka/secrets/keystore_creds:ro,z
- ./testHelpers/certs/sslkey_creds:/etc/kafka/secrets/sslkey_creds:ro,z
- ./testHelpers/certs/truststore_creds:/etc/kafka/secrets/truststore_creds:ro,z
- ./testHelpers/kafka/server-jaas.conf:/opt/kafka/config/server-jaas.conf:ro,z

kafka2:
image: confluentinc/cp-kafka:5.4.2
hostname: kafka2
container_name: kafka2
labels:
- 'custom.project=kafkajs'
- 'custom.service=kafka2'
depends_on:
- zookeeper
ports:
- '29095:29095'
- '9095:9095'
- '29096:29096'
- '9096:9096'
- '29097:29097'
- '9097:9097'
environment:
KAFKA_BROKER_ID: '1'
KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT,SSL:SSL,SSL_HOST:SSL,SASL_SSL:SASL_SSL,SASL_SSL_HOST:SASL_SSL
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka2:29095,PLAINTEXT_HOST://localhost:9095,SSL://kafka2:29096,SSL_HOST://localhost:9096,SASL_SSL://kafka2:29097,SASL_SSL_HOST://localhost:9097
KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
KAFKA_DELETE_TOPIC_ENABLE: 'true'
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: '0'
KAFKA_SSL_KEYSTORE_FILENAME: 'kafka.server.keystore.jks'
KAFKA_SSL_KEYSTORE_CREDENTIALS: 'keystore_creds'
KAFKA_SSL_KEY_CREDENTIALS: 'sslkey_creds'
KAFKA_SSL_TRUSTSTORE_FILENAME: 'kafka.server.truststore.jks'
KAFKA_SSL_TRUSTSTORE_CREDENTIALS: 'truststore_creds'
KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL: 'PLAIN'
KAFKA_SASL_ENABLED_MECHANISMS: 'PLAIN,SCRAM-SHA-256,SCRAM-SHA-512'
KAFKA_OPTS: '-Djava.security.auth.login.config=/opt/kafka/config/server-jaas.conf'
# suppress verbosity
# https://github.com/confluentinc/cp-docker-images/blob/master/debian/kafka/include/etc/confluent/docker/log4j.properties.template
KAFKA_LOG4J_LOGGERS: 'kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO'
volumes:
- ./testHelpers/certs/kafka.server.keystore.jks:/etc/kafka/secrets/kafka.server.keystore.jks:ro,z
- ./testHelpers/certs/kafka.server.truststore.jks:/etc/kafka/secrets/kafka.server.truststore.jks:ro,z
- ./testHelpers/certs/keystore_creds:/etc/kafka/secrets/keystore_creds:ro,z
- ./testHelpers/certs/sslkey_creds:/etc/kafka/secrets/sslkey_creds:ro,z
- ./testHelpers/certs/truststore_creds:/etc/kafka/secrets/truststore_creds:ro,z
- ./testHelpers/kafka/server-jaas.conf:/opt/kafka/config/server-jaas.conf:ro,z

kafka3:
image: confluentinc/cp-kafka:5.4.2
hostname: kafka3
container_name: kafka3
labels:
- 'custom.project=kafkajs'
- 'custom.service=kafka3'
depends_on:
- zookeeper
ports:
- '29098:29098'
- '9098:9098'
- '29099:29099'
- '9099:9099'
- '29100:29100'
- '9100:9100'
environment:
KAFKA_BROKER_ID: '2'
KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT,SSL:SSL,SSL_HOST:SSL,SASL_SSL:SASL_SSL,SASL_SSL_HOST:SASL_SSL
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka3:29098,PLAINTEXT_HOST://localhost:9098,SSL://kafka3:29099,SSL_HOST://localhost:9099,SASL_SSL://kafka3:29100,SASL_SSL_HOST://localhost:9100
KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
KAFKA_DELETE_TOPIC_ENABLE: 'true'
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: '0'
KAFKA_SSL_KEYSTORE_FILENAME: 'kafka.server.keystore.jks'
KAFKA_SSL_KEYSTORE_CREDENTIALS: 'keystore_creds'
KAFKA_SSL_KEY_CREDENTIALS: 'sslkey_creds'
KAFKA_SSL_TRUSTSTORE_FILENAME: 'kafka.server.truststore.jks'
KAFKA_SSL_TRUSTSTORE_CREDENTIALS: 'truststore_creds'
KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL: 'PLAIN'
KAFKA_SASL_ENABLED_MECHANISMS: 'PLAIN,SCRAM-SHA-256,SCRAM-SHA-512'
KAFKA_OPTS: '-Djava.security.auth.login.config=/opt/kafka/config/server-jaas.conf'
# suppress verbosity
# https://github.com/confluentinc/cp-docker-images/blob/master/debian/kafka/include/etc/confluent/docker/log4j.properties.template
KAFKA_LOG4J_LOGGERS: 'kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO'
volumes:
- ./testHelpers/certs/kafka.server.keystore.jks:/etc/kafka/secrets/kafka.server.keystore.jks:ro,z
- ./testHelpers/certs/kafka.server.truststore.jks:/etc/kafka/secrets/kafka.server.truststore.jks:ro,z
- ./testHelpers/certs/keystore_creds:/etc/kafka/secrets/keystore_creds:ro,z
- ./testHelpers/certs/sslkey_creds:/etc/kafka/secrets/sslkey_creds:ro,z
- ./testHelpers/certs/truststore_creds:/etc/kafka/secrets/truststore_creds:ro,z
- ./testHelpers/kafka/server-jaas.conf:/opt/kafka/config/server-jaas.conf:ro,z
143 changes: 143 additions & 0 deletions docker-compose.2_4_oauthbearer.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,143 @@
version: '2'
services:
zookeeper:
image: confluentinc/cp-zookeeper:latest
hostname: zookeeper
container_name: zookeeper
ports:
- '2181:2181'
environment:
ZOOKEEPER_CLIENT_PORT: '2181'
ZOOKEEPER_TICK_TIME: '2000'
KAFKA_OPTS: '-Djava.security.auth.login.config=/etc/kafka/server-jaas.conf -Dzookeeper.authProvider.1=org.apache.zookeeper.server.auth.SASLAuthenticationProvider'
volumes:
- ./testHelpers/kafka/server-jaas_oauth.conf:/etc/kafka/server-jaas.conf:ro,z

kafka1:
image: confluentinc/cp-kafka:5.4.2
hostname: kafka1
container_name: kafka1
labels:
- 'custom.project=kafkajs'
- 'custom.service=kafka1'
depends_on:
- zookeeper
ports:
- '29092:29092'
- '9092:9092'
- '29093:29093'
- '9093:9093'
- '29094:29094'
- '9094:9094'
environment:
KAFKA_BROKER_ID: '0'
KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT,SSL:SSL,SSL_HOST:SSL,SASL_SSL:SASL_SSL,SASL_SSL_HOST:SASL_SSL
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka1:29092,PLAINTEXT_HOST://localhost:9092,SSL://kafka1:29093,SSL_HOST://localhost:9093,SASL_SSL://kafka1:29094,SASL_SSL_HOST://localhost:9094
KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
KAFKA_DELETE_TOPIC_ENABLE: 'true'
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: '0'
KAFKA_SSL_KEYSTORE_FILENAME: 'kafka.server.keystore.jks'
KAFKA_SSL_KEYSTORE_CREDENTIALS: 'keystore_creds'
KAFKA_SSL_KEY_CREDENTIALS: 'sslkey_creds'
KAFKA_SSL_TRUSTSTORE_FILENAME: 'kafka.server.truststore.jks'
KAFKA_SSL_TRUSTSTORE_CREDENTIALS: 'truststore_creds'
KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL: 'PLAIN'
KAFKA_SASL_ENABLED_MECHANISMS: 'OAUTHBEARER'
KAFKA_OPTS: '-Djava.security.auth.login.config=/opt/kafka/config/server-jaas.conf'
# suppress verbosity
# https://github.com/confluentinc/cp-docker-images/blob/master/debian/kafka/include/etc/confluent/docker/log4j.properties.template
KAFKA_LOG4J_LOGGERS: 'kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO'
volumes:
- ./testHelpers/certs/kafka.server.keystore.jks:/etc/kafka/secrets/kafka.server.keystore.jks:ro,z
- ./testHelpers/certs/kafka.server.truststore.jks:/etc/kafka/secrets/kafka.server.truststore.jks:ro,z
- ./testHelpers/certs/keystore_creds:/etc/kafka/secrets/keystore_creds:ro,z
- ./testHelpers/certs/sslkey_creds:/etc/kafka/secrets/sslkey_creds:ro,z
- ./testHelpers/certs/truststore_creds:/etc/kafka/secrets/truststore_creds:ro,z
- ./testHelpers/kafka/server-jaas_oauth.conf:/opt/kafka/config/server-jaas.conf:ro,z

kafka2:
image: confluentinc/cp-kafka:5.4.2
hostname: kafka2
container_name: kafka2
labels:
- 'custom.project=kafkajs'
- 'custom.service=kafka2'
depends_on:
- zookeeper
ports:
- '29095:29095'
- '9095:9095'
- '29096:29096'
- '9096:9096'
- '29097:29097'
- '9097:9097'
environment:
KAFKA_BROKER_ID: '1'
KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT,SSL:SSL,SSL_HOST:SSL,SASL_SSL:SASL_SSL,SASL_SSL_HOST:SASL_SSL
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka2:29095,PLAINTEXT_HOST://localhost:9095,SSL://kafka2:29096,SSL_HOST://localhost:9096,SASL_SSL://kafka2:29097,SASL_SSL_HOST://localhost:9097
KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
KAFKA_DELETE_TOPIC_ENABLE: 'true'
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: '0'
KAFKA_SSL_KEYSTORE_FILENAME: 'kafka.server.keystore.jks'
KAFKA_SSL_KEYSTORE_CREDENTIALS: 'keystore_creds'
KAFKA_SSL_KEY_CREDENTIALS: 'sslkey_creds'
KAFKA_SSL_TRUSTSTORE_FILENAME: 'kafka.server.truststore.jks'
KAFKA_SSL_TRUSTSTORE_CREDENTIALS: 'truststore_creds'
KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL: 'PLAIN'
KAFKA_SASL_ENABLED_MECHANISMS: 'OAUTHBEARER'
KAFKA_OPTS: '-Djava.security.auth.login.config=/opt/kafka/config/server-jaas.conf'
# suppress verbosity
# https://github.com/confluentinc/cp-docker-images/blob/master/debian/kafka/include/etc/confluent/docker/log4j.properties.template
KAFKA_LOG4J_LOGGERS: 'kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO'
volumes:
- ./testHelpers/certs/kafka.server.keystore.jks:/etc/kafka/secrets/kafka.server.keystore.jks:ro,z
- ./testHelpers/certs/kafka.server.truststore.jks:/etc/kafka/secrets/kafka.server.truststore.jks:ro,z
- ./testHelpers/certs/keystore_creds:/etc/kafka/secrets/keystore_creds:ro,z
- ./testHelpers/certs/sslkey_creds:/etc/kafka/secrets/sslkey_creds:ro,z
- ./testHelpers/certs/truststore_creds:/etc/kafka/secrets/truststore_creds:ro,z
- ./testHelpers/kafka/server-jaas_oauth.conf:/opt/kafka/config/server-jaas.conf:ro,z

kafka3:
image: confluentinc/cp-kafka:5.4.2
hostname: kafka3
container_name: kafka3
labels:
- 'custom.project=kafkajs'
- 'custom.service=kafka3'
depends_on:
- zookeeper
ports:
- '29098:29098'
- '9098:9098'
- '29099:29099'
- '9099:9099'
- '29100:29100'
- '9100:9100'
environment:
KAFKA_BROKER_ID: '2'
KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT,SSL:SSL,SSL_HOST:SSL,SASL_SSL:SASL_SSL,SASL_SSL_HOST:SASL_SSL
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka3:29098,PLAINTEXT_HOST://localhost:9098,SSL://kafka3:29099,SSL_HOST://localhost:9099,SASL_SSL://kafka3:29100,SASL_SSL_HOST://localhost:9100
KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
KAFKA_DELETE_TOPIC_ENABLE: 'true'
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: '0'
KAFKA_SSL_KEYSTORE_FILENAME: 'kafka.server.keystore.jks'
KAFKA_SSL_KEYSTORE_CREDENTIALS: 'keystore_creds'
KAFKA_SSL_KEY_CREDENTIALS: 'sslkey_creds'
KAFKA_SSL_TRUSTSTORE_FILENAME: 'kafka.server.truststore.jks'
KAFKA_SSL_TRUSTSTORE_CREDENTIALS: 'truststore_creds'
KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL: 'PLAIN'
KAFKA_SASL_ENABLED_MECHANISMS: 'OAUTHBEARER'
KAFKA_OPTS: '-Djava.security.auth.login.config=/opt/kafka/config/server-jaas.conf'
# suppress verbosity
# https://github.com/confluentinc/cp-docker-images/blob/master/debian/kafka/include/etc/confluent/docker/log4j.properties.template
KAFKA_LOG4J_LOGGERS: 'kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO'
volumes:
- ./testHelpers/certs/kafka.server.keystore.jks:/etc/kafka/secrets/kafka.server.keystore.jks:ro,z
- ./testHelpers/certs/kafka.server.truststore.jks:/etc/kafka/secrets/kafka.server.truststore.jks:ro,z
- ./testHelpers/certs/keystore_creds:/etc/kafka/secrets/keystore_creds:ro,z
- ./testHelpers/certs/sslkey_creds:/etc/kafka/secrets/sslkey_creds:ro,z
- ./testHelpers/certs/truststore_creds:/etc/kafka/secrets/truststore_creds:ro,z
- ./testHelpers/kafka/server-jaas_oauth.conf:/opt/kafka/config/server-jaas.conf:ro,z
35 changes: 35 additions & 0 deletions docs/Configuration.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,41 @@ new Kafka({
})
```

### OAUTHBEARER Example

```javascript
new Kafka({
clientId: 'my-app',
brokers: ['kafka1:9092', 'kafka2:9092'],
// authenticationTimeout: 1000,
// reauthenticationThreshold: 10000,
ssl: true,
sasl: {
mechanism: 'oauthbearer',
oauthBearerProvider: async () => {
// Use an unsecured token...
const token = jwt.sign({ sub: 'test' }, 'abc', { algorithm: 'none' })

// ...or, more realistically, grab the token from some OAuth endpoint

return {
value: token
}
}
},
})
```

The `sasl` object must include a property named `oauthBearerProvider`, an
async function that is used to return the OAuth bearer token.

The OAuth bearer token must be an object with properties value and
(optionally) extensions, that will be sent during the SASL/OAUTHBEARER
request.

The implementation of the oauthBearerProvider must take care that tokens are
reused and refreshed when appropriate.

### AWS IAM Example

```javascript
Expand Down
2 changes: 1 addition & 1 deletion docs/DockerLocal.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,4 +45,4 @@ You will now be able to connect to your Kafka broker at `$(HOST_IP):9092`. See t

## SSL & authentication methods

To configure Kafka to use SSL and/or authentication methods such as SASL, see [docker-compose.yml](https://github.com/tulios/kafkajs/blob/master/docker-compose.2_3.yml). This configuration is used while developing KafkaJS, and is more complicated to set up, but may give you a more production-like development environment.
To configure Kafka to use SSL and/or authentication methods such as SASL, see [docker-compose.yml](https://github.com/tulios/kafkajs/blob/master/docker-compose.2_4.yml). This configuration is used while developing KafkaJS, and is more complicated to set up, but may give you a more production-like development environment.
Loading

0 comments on commit b688f0e

Please sign in to comment.