Skip to content

Commit f16a3ea

Browse files
authored
[RORDEV-1453] Ubuntu apt-get based installation of ES in tests (#1127)
1 parent 1e3bb21 commit f16a3ea

19 files changed

+459
-76
lines changed

ci/run-pipeline.sh

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -54,8 +54,7 @@ run_integration_tests() {
5454
ES_MODULE=$1
5555

5656
echo ">>> $ES_MODULE => Running testcontainers.."
57-
./gradlew ror-tools:test "-PesModule=$ES_MODULE" || (find . | grep hs_err | xargs cat && exit 1)
58-
./gradlew integration-tests:test "-PesModule=$ES_MODULE" || (find . | grep hs_err | xargs cat && exit 1)
57+
./gradlew --no-daemon ror-tools:test integration-tests:test "-PesModule=$ES_MODULE" || (find . | grep hs_err | xargs cat && exit 1)
5958
}
6059

6160
if [[ -z $TRAVIS ]] || [[ $ROR_TASK == "integration_es90x" ]]; then
Lines changed: 191 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,191 @@
1+
/*
2+
* This file is part of ReadonlyREST.
3+
*
4+
* ReadonlyREST is free software: you can redistribute it and/or modify
5+
* it under the terms of the GNU General Public License as published by
6+
* the Free Software Foundation, either version 3 of the License, or
7+
* (at your option) any later version.
8+
*
9+
* ReadonlyREST is distributed in the hope that it will be useful,
10+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+
* GNU General Public License for more details.
13+
*
14+
* You should have received a copy of the GNU General Public License
15+
* along with ReadonlyREST. If not, see http://www.gnu.org/licenses/
16+
*/
17+
package tech.beshu.ror.tools
18+
19+
import cats.data.NonEmptyList
20+
import com.github.dockerjava.api.DockerClient
21+
import monix.eval.Task
22+
import monix.execution.Scheduler
23+
import monix.execution.atomic.AtomicInt
24+
import org.scalatest.matchers.must.Matchers.include
25+
import org.scalatest.matchers.should.Matchers.{should, shouldNot}
26+
import org.scalatest.wordspec.AnyWordSpec
27+
import org.testcontainers.DockerClientFactory
28+
import tech.beshu.ror.integration.utils.ESVersionSupportForAnyWordSpecLike
29+
import tech.beshu.ror.utils.containers.*
30+
import tech.beshu.ror.utils.containers.EsContainerCreator.EsNodeSettings
31+
import tech.beshu.ror.utils.containers.images.Elasticsearch.EsInstallationType
32+
import tech.beshu.ror.utils.containers.images.ReadonlyRestWithEnabledXpackSecurityPlugin
33+
import tech.beshu.ror.utils.containers.logs.DockerLogsToStringConsumer
34+
import tech.beshu.ror.utils.elasticsearch.BaseManager.JSON
35+
import tech.beshu.ror.utils.elasticsearch.SearchManager
36+
import tech.beshu.ror.utils.httpclient.RestClient
37+
import tech.beshu.ror.utils.misc.EsModulePatterns
38+
39+
import scala.concurrent.duration.*
40+
import scala.language.postfixOps
41+
import scala.util.Try
42+
43+
// There is a change introduced in Elasticsearch since versions 9.0.1 and 8.18.1 (older ES versions are not affected)
44+
// The change: https://github.com/elastic/elasticsearch/pull/126852 ("With this PR we restrict the paths we allow access to, forbidding plugins to specify/request entitlements for reading or writing to specific protected directories.")
45+
// In our use case it causes problems with apt-based installations of ES and patching:
46+
// - ROR cannot check (on startup) whether the ES is patched
47+
// - that is because after the aforementioned change in ES, the ROR plugin cannot access the /usr/share/elasticsearch directory
48+
// - we bypass this problem (ROR plugin cannot check the patch status, so it just allows to continue starting ES, with warning in logs)
49+
// This test suite verifies, that both official ES image and apt-based ES installation with ROR can start. Logs are also asserted to detect the warning.
50+
class PatchingOfAptBasedEsInstallationSuite extends AnyWordSpec with ESVersionSupportForAnyWordSpecLike {
51+
52+
import PatchingOfAptBasedEsInstallationSuite.*
53+
54+
implicit val scheduler: Scheduler = Scheduler.computation(10)
55+
56+
private val validRorConfigFile = "/basic/readonlyrest.yml"
57+
58+
"ES" when {
59+
"using official ES image" should {
60+
"successfully load ROR plugin and start (patch verification without warning)" in {
61+
val dockerLogs = withTestEsContainerManager(EsInstallationType.EsDockerImage) { esContainer =>
62+
testRorStartup(usingManager = esContainer)
63+
}
64+
dockerLogs should include("ReadonlyREST is waiting for full Elasticsearch init")
65+
dockerLogs should include("Elasticsearch fully initiated. ReadonlyREST can continue ...")
66+
dockerLogs should include("Loading Elasticsearch settings from file: /usr/share/elasticsearch/config/elasticsearch.yml")
67+
dockerLogs shouldNot include("Cannot verify if the ES was patched")
68+
dockerLogs should include("ReadonlyREST was loaded")
69+
}
70+
}
71+
"installed on Ubuntu using apt" should {
72+
// ES 6.x is not available as apt package, so we do not test it
73+
"ES {7.x, 8.0.x - 8.17.x} successfully load ROR plugin and start (without warning about not being able to verify patch)" excludeES(allEs6x, allEs9x, allEs818x) in {
74+
val dockerLogs = withTestEsContainerManager(EsInstallationType.UbuntuDockerImageWithEsFromApt) { esContainer =>
75+
testRorStartup(usingManager = esContainer)
76+
}
77+
dockerLogs should include("ReadonlyREST is waiting for full Elasticsearch init")
78+
dockerLogs should include("Elasticsearch fully initiated. ReadonlyREST can continue ...")
79+
dockerLogs should include("Loading Elasticsearch settings from file: /etc/elasticsearch/elasticsearch.yml")
80+
dockerLogs shouldNot include("Cannot verify if the ES was patched")
81+
dockerLogs should include("ReadonlyREST was loaded")
82+
}
83+
"ES {8.18.x, 9.x} successfully load ROR plugin and start (with warning about not being able to verify patch)" excludeES(allEs6x, allEs7x, allEs8xBelowEs818x) in {
84+
val dockerLogs = withTestEsContainerManager(EsInstallationType.UbuntuDockerImageWithEsFromApt) { esContainer =>
85+
testRorStartup(usingManager = esContainer)
86+
}
87+
dockerLogs should include("ReadonlyREST is waiting for full Elasticsearch init")
88+
dockerLogs should include("Elasticsearch fully initiated. ReadonlyREST can continue ...")
89+
dockerLogs should include("Loading Elasticsearch settings from file: /etc/elasticsearch/elasticsearch.yml")
90+
dockerLogs should include("Cannot verify if the ES was patched. component [readonlyrest], module [ALL-UNNAMED], class [class tech.beshu.ror.tools.core.utils.EsDirectory$], entitlement [file], operation [read], path [/usr/share/elasticsearch]")
91+
dockerLogs should include("ReadonlyREST was loaded")
92+
}
93+
}
94+
}
95+
96+
private def withTestEsContainerManager(esInstallationType: EsInstallationType)
97+
(testCode: TestEsContainerManager => Task[Unit]): String = {
98+
val esContainer = new TestEsContainerManager(validRorConfigFile, esInstallationType)
99+
try {
100+
(for {
101+
_ <- esContainer.start()
102+
_ <- testCode(esContainer)
103+
} yield ()).runSyncUnsafe(5 minutes)
104+
esContainer.getLogs
105+
} finally {
106+
esContainer.stop().runSyncUnsafe()
107+
}
108+
}
109+
110+
private def testRorStartup(usingManager: TestEsContainerManager): Task[Unit] = {
111+
for {
112+
restClient <- usingManager.createRestClient
113+
searchTestResults <- searchTest(restClient)
114+
result <- handleResult(searchTestResults)
115+
} yield result
116+
}
117+
118+
private def searchTest(client: RestClient): Task[TestResponse] = Task.delay {
119+
val manager = new SearchManager(client, esVersionUsed)
120+
val response = manager.searchAll("*")
121+
TestResponse(response.responseCode, response.responseJson)
122+
}
123+
124+
private def handleResult(result: TestResponse): Task[Unit] = {
125+
val hasEsRespondedWithSuccess = result.responseCode == 200
126+
if (hasEsRespondedWithSuccess) {
127+
Task.unit
128+
} else {
129+
Task.raiseError(new IllegalStateException(s"Test failed. Expected success response but was: [$result]"))
130+
}
131+
}
132+
}
133+
134+
private object PatchingOfAptBasedEsInstallationSuite extends EsModulePatterns {
135+
final case class TestResponse(responseCode: Int, responseJson: JSON)
136+
137+
private val uniqueClusterId: AtomicInt = AtomicInt(1)
138+
139+
final class TestEsContainerManager(rorConfigFile: String, esInstallationType: EsInstallationType) extends EsContainerCreator {
140+
141+
private val dockerClient: DockerClient = DockerClientFactory.instance().client()
142+
143+
private val dockerLogsCollector = new DockerLogsToStringConsumer
144+
145+
private val esContainer = createEsContainer
146+
147+
def start(): Task[Unit] = Task.delay(esContainer.start())
148+
149+
def stop(): Task[Unit] = for {
150+
_ <- Task.delay(esContainer.stop())
151+
_ <- Task.delay(dockerClient.removeImageCmd(esContainer.imageFromDockerfile.get()).withForce(true).exec())
152+
} yield ()
153+
154+
def getLogs: String = dockerLogsCollector.getLogs
155+
156+
def createRestClient: Task[RestClient] = {
157+
Task.tailRecM(()) { _ =>
158+
Task.delay(createAdminClient)
159+
}
160+
}
161+
162+
private def createAdminClient = {
163+
Try(esContainer.adminClient)
164+
.toEither
165+
.left.map(_ => ())
166+
}
167+
168+
private def createEsContainer: EsContainer = {
169+
val clusterName = s"ROR_${uniqueClusterId.getAndIncrement()}"
170+
val nodeName = s"${clusterName}_1"
171+
create(
172+
nodeSettings = EsNodeSettings(
173+
nodeName = nodeName,
174+
clusterName = clusterName,
175+
securityType = SecurityType.RorWithXpackSecurity(
176+
ReadonlyRestWithEnabledXpackSecurityPlugin.Config.Attributes.default.copy(
177+
rorConfigFileName = rorConfigFile
178+
)
179+
),
180+
containerSpecification = ContainerSpecification.empty,
181+
esVersion = EsVersion.DeclaredInProject
182+
),
183+
allNodeNames = NonEmptyList.of(nodeName),
184+
nodeDataInitializer = NoOpElasticsearchNodeDataInitializer,
185+
startedClusterDependencies = StartedClusterDependencies(List.empty),
186+
esInstallationType = esInstallationType,
187+
additionalLogConsumer = Some(dockerLogsCollector)
188+
)
189+
}
190+
}
191+
}

ror-tools/src/test/scala/tech/beshu/ror/tools/utils/ExampleEsWithRorContainer.scala

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ import tech.beshu.ror.utils.containers.*
2626
import tech.beshu.ror.utils.containers.ElasticsearchNodeWaitingStrategy.AwaitingReadyStrategy
2727
import tech.beshu.ror.utils.containers.EsContainerCreator.EsNodeSettings
2828
import tech.beshu.ror.utils.containers.exceptions.ContainerCreationException
29+
import tech.beshu.ror.utils.containers.images.Elasticsearch.EsInstallationType
2930
import tech.beshu.ror.utils.containers.images.domain.Enabled
3031
import tech.beshu.ror.utils.containers.images.{Elasticsearch, ReadonlyRestWithEnabledXpackSecurityPlugin}
3132
import tech.beshu.ror.utils.gradle.RorPluginGradleProject
@@ -96,7 +97,8 @@ class ExampleEsWithRorContainer(implicit scheduler: Scheduler) extends EsContain
9697
nodeName = nodeSettings.nodeName,
9798
masterNodes = allNodeNames,
9899
additionalElasticsearchYamlEntries = nodeSettings.containerSpecification.additionalElasticsearchYamlEntries,
99-
envs = nodeSettings.containerSpecification.environmentVariables
100+
envs = nodeSettings.containerSpecification.environmentVariables,
101+
esInstallationType = EsInstallationType.EsDockerImage,
100102
),
101103
securityConfig = ReadonlyRestWithEnabledXpackSecurityPlugin.Config(
102104
rorPlugin = pluginFile.toScala,
@@ -107,6 +109,7 @@ class ExampleEsWithRorContainer(implicit scheduler: Scheduler) extends EsContain
107109
startedClusterDependencies = startedClusterDependencies,
108110
customEntrypoint = Some(Path("""/bin/sh -c "while true; do sleep 30; done"""")),
109111
awaitingReadyStrategy = AwaitingReadyStrategy.ImmediatelyTreatAsReady,
112+
additionalLogConsumer = None,
110113
)
111114
}
112115
}

tests-utils/src/main/scala/tech/beshu/ror/utils/containers/EsClusterProvider.scala

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,12 @@ trait EsClusterProvider extends EsContainerCreator with EsModulePatterns {
6666
private def nodeCreator(nodeSettings: EsNodeSettings,
6767
allNodeNames: NonEmptyList[String],
6868
nodeDataInitializer: ElasticsearchNodeDataInitializer): StartedClusterDependencies => EsContainer = { deps =>
69-
this.create(nodeSettings, allNodeNames, nodeDataInitializer, deps)
69+
this.create(
70+
nodeSettings = nodeSettings,
71+
allNodeNames = allNodeNames,
72+
nodeDataInitializer = nodeDataInitializer,
73+
startedClusterDependencies = deps,
74+
additionalLogConsumer = None,
75+
)
7076
}
7177
}

tests-utils/src/main/scala/tech/beshu/ror/utils/containers/EsContainer.scala

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ import tech.beshu.ror.utils.containers.ElasticsearchNodeWaitingStrategy.Awaiting
2727
import tech.beshu.ror.utils.containers.EsContainer.Credentials
2828
import tech.beshu.ror.utils.containers.EsContainer.Credentials.{BasicAuth, Header, None, Token}
2929
import tech.beshu.ror.utils.containers.images.Elasticsearch
30+
import tech.beshu.ror.utils.containers.logs.CompositeLogConsumer
3031
import tech.beshu.ror.utils.containers.providers.ClientProvider
3132
import tech.beshu.ror.utils.httpclient.RestClient
3233
import tech.beshu.ror.utils.misc.ScalaUtils.finiteDurationToJavaDuration
@@ -39,12 +40,12 @@ import scala.language.postfixOps
3940
abstract class EsContainer(val esVersion: String,
4041
val esConfig: Elasticsearch.Config,
4142
val startedClusterDependencies: StartedClusterDependencies,
42-
image: ImageFromDockerfile)
43+
val imageFromDockerfile: ImageFromDockerfile)
4344
extends SingleContainer[GenericContainer[_]]
4445
with ClientProvider
4546
with StrictLogging {
4647

47-
override implicit val container: GenericContainer[_] = new org.testcontainers.containers.GenericContainer(image)
48+
override implicit val container: GenericContainer[_] = new org.testcontainers.containers.GenericContainer(imageFromDockerfile)
4849

4950
def sslEnabled: Boolean
5051

@@ -67,8 +68,13 @@ object EsContainer {
6768
def init(esContainer: EsContainer,
6869
initializer: ElasticsearchNodeDataInitializer,
6970
logger: Logger,
71+
additionalLogConsumer: Option[Consumer[OutputFrame]],
7072
awaitingReadyStrategy: AwaitingReadyStrategy = AwaitingReadyStrategy.WaitForEsReadiness): EsContainer = {
71-
val logConsumer: Consumer[OutputFrame] = new Slf4jLogConsumer(logger.underlying)
73+
val slf4jConsumer = new Slf4jLogConsumer(logger.underlying)
74+
val logConsumer: Consumer[OutputFrame] = additionalLogConsumer match {
75+
case Some(additional) => new CompositeLogConsumer(slf4jConsumer, additional)
76+
case scala.None => slf4jConsumer
77+
}
7278
val esClient = Coeval(esContainer.adminClient)
7379
esContainer.container.setLogConsumers((logConsumer :: Nil).asJava)
7480
esContainer.container.addExposedPort(9200)

0 commit comments

Comments
 (0)