Crawling Azure devops repositories

github-crawler-autoconfigure/pom.xml

@@ -37,6 +37,13 @@
             </exclusions>
         </dependency>
 
+        <dependency>
+            <groupId>com.github.tomakehurst</groupId>
+            <artifactId>wiremock-jre8</artifactId>
+            <version>2.32.0</version>
+            <scope>test</scope>
+        </dependency>
+
 
         <dependency>
             <groupId>com.jayway.awaitility</groupId>

github-crawler-autoconfigure/src/main/kotlin/com/societegenerale/githubcrawler/config/AzureDevopsConfiguration.kt

@@ -0,0 +1,38 @@
+package com.societegenerale.githubcrawler.config
+
+
+import com.societegenerale.githubcrawler.GitHubCrawlerProperties
+import com.societegenerale.githubcrawler.remote.RemoteAzureDevopsImpl
+import com.societegenerale.githubcrawler.remote.RemoteAzureDevopsImpl.Companion.AZURE_DEVOPS_URL
+import com.societegenerale.githubcrawler.remote.RemoteGitHub
+import org.slf4j.LoggerFactory
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty
+import org.springframework.context.annotation.Bean
+import org.springframework.context.annotation.Configuration
+
+@Configuration
+@ConditionalOnProperty(prefix = "github-crawler.githubConfig", name = ["type"], havingValue = "AZURE_DEVOPS")
+open class AzureDevopsConfiguration {
+
+    val log = LoggerFactory.getLogger(this.javaClass)
+
+    @Bean
+    open fun remoteAzureDevops(gitHubCrawlerProperties: GitHubCrawlerProperties): RemoteGitHub {
+
+        val targetUrl:String
+
+        if(gitHubCrawlerProperties.githubConfig.apiUrl.isEmpty()){
+            targetUrl=  AZURE_DEVOPS_URL
+        }
+        else{
+            targetUrl=gitHubCrawlerProperties.githubConfig.apiUrl
+        }
+
+        log.info("URL for AzureDevops : $targetUrl")
+
+        return RemoteAzureDevopsImpl(targetUrl,
+                                     gitHubCrawlerProperties.githubConfig.organizationName,
+                                     gitHubCrawlerProperties.githubConfig.oauthToken)
+    }
+
+}

github-crawler-autoconfigure/src/main/kotlin/com/societegenerale/githubcrawler/config/GitHubCrawlerAutoConfiguration.kt

@@ -1,29 +1,27 @@
 package com.societegenerale.githubcrawler.config
 
 
-import com.societegenerale.githubcrawler.ConfigValidator
-import com.societegenerale.githubcrawler.GitHubCrawler
-import com.societegenerale.githubcrawler.GitHubCrawlerProperties
-import com.societegenerale.githubcrawler.RepositoryEnricher
+import com.societegenerale.githubcrawler.*
 import com.societegenerale.githubcrawler.output.GitHubCrawlerOutput
 import com.societegenerale.githubcrawler.parsers.FileContentParser
 import com.societegenerale.githubcrawler.remote.RemoteGitHub
-import com.societegenerale.githubcrawler.remote.RemoteGitHubImpl
-import com.societegenerale.githubcrawler.remote.RemoteGitLabImpl
 import com.societegenerale.githubcrawler.repoTaskToPerform.RepoTaskBuilder
+import org.slf4j.LoggerFactory
 import org.springframework.boot.context.properties.EnableConfigurationProperties
 import org.springframework.context.annotation.Bean
 import org.springframework.context.annotation.Configuration
 import org.springframework.context.annotation.Import
-import org.springframework.context.annotation.Profile
 import org.springframework.core.convert.ConversionService
 import org.springframework.core.env.Environment
 
 @Configuration
-@Import(GitHubCrawlerParserConfig::class,GitHubCrawlerOutputConfig::class,GitHubCrawlerMiscTasksConfig::class,GitHubConfiguration::class,GitLabConfiguration::class)
+@Import(GitHubCrawlerParserConfig::class,GitHubCrawlerOutputConfig::class,GitHubCrawlerMiscTasksConfig::class,
+    GitHubConfiguration::class,GitLabConfiguration::class,AzureDevopsConfiguration::class)
 @EnableConfigurationProperties(GitHubCrawlerProperties::class)
 open class GitHubCrawlerAutoConfiguration {
 
+    val log = LoggerFactory.getLogger(this.javaClass)
+
     @Bean
     open fun conversionService(): ConversionService {
         //TODO custom converter should be an annotation if possible - right now, it works because we instantiate a bean named conversionService
@@ -38,12 +36,16 @@ open class GitHubCrawlerAutoConfiguration {
                      environment : Environment,
                      configValidator: ConfigValidator,
                      fileContentParsers: List<FileContentParser>,
-                     repoTasksBuilder: List<RepoTaskBuilder>
+                     repoTasksBuilders: List<RepoTaskBuilder>
                      ): GitHubCrawler {
 
-        val repositoryEnricher = RepositoryEnricher(remoteGitHub)
+        var availableParsersAndTasks = AvailableParsersAndTasks(fileContentParsers,repoTasksBuilders)
+
+        val repositoryEnricher = RepositoryEnricher(remoteGitHub,availableParsersAndTasks)
+
+        log.info("using repositoryEnricher "+repositoryEnricher+" when building the crawler...")
 
-        return GitHubCrawler(remoteGitHub, output, repositoryEnricher,gitHubCrawlerProperties,environment,gitHubCrawlerProperties.githubConfig.organizationName,configValidator,fileContentParsers,repoTasksBuilder)
+        return GitHubCrawler(remoteGitHub, output, repositoryEnricher,gitHubCrawlerProperties,environment,gitHubCrawlerProperties.githubConfig.organizationName,configValidator,availableParsersAndTasks)
     }
 
     @Bean

github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/AzureDevopsCrawlerIT.java

@@ -0,0 +1,115 @@
+package com.societegenerale.githubcrawler;
+
+import static com.github.tomakehurst.wiremock.client.WireMock.aResponse;
+import static com.github.tomakehurst.wiremock.client.WireMock.configureFor;
+import static com.github.tomakehurst.wiremock.client.WireMock.stubFor;
+import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo;
+import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.options;
+
+import com.github.tomakehurst.wiremock.WireMockServer;
+import com.github.tomakehurst.wiremock.client.WireMock;
+import com.societegenerale.githubcrawler.config.GitHubCrawlerAutoConfiguration;
+import com.societegenerale.githubcrawler.config.TestConfig;
+import com.societegenerale.githubcrawler.remote.RemoteGitHub;
+import java.io.IOException;
+import org.apache.commons.io.FileUtils;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.context.ApplicationContext;
+import org.springframework.test.annotation.DirtiesContext;
+import org.springframework.test.annotation.DirtiesContext.ClassMode;
+import org.springframework.test.context.ActiveProfiles;
+import org.springframework.util.ResourceUtils;
+
+
+@SpringBootTest(classes = {TestConfig.class, GitHubCrawlerAutoConfiguration.class})
+@ActiveProfiles(profiles = {"azureDevopsTest"})
+@DirtiesContext(classMode = ClassMode.BEFORE_CLASS)
+class AzureDevopsCrawlerIT {
+
+  Logger log = LoggerFactory.getLogger(this.getClass().toString());
+
+  private final static String API_VERSION = "api-version=7.1-preview.1";
+
+  private final static int AZUREDEVOPS_SERVER_PORT_FOR_TESTS=9901;
+
+  @Autowired
+  ApplicationContext context;
+
+  @Autowired
+  private GitHubCrawler crawler;
+
+  private final WireMockServer wm = new WireMockServer(
+      options()
+          .port(AZUREDEVOPS_SERVER_PORT_FOR_TESTS)
+          .usingFilesUnderDirectory("src/test/resources/azureDevops"));
+
+  @BeforeEach
+  void mockSetUp() throws IOException {
+
+    wm.start();
+    configureFor(AZUREDEVOPS_SERVER_PORT_FOR_TESTS);
+
+    stubFor(WireMock.get(urlEqualTo("/platform/platform-projects/_apis/git/repositories?"+API_VERSION))
+        .willReturn(aResponse()
+            .withBodyFile("repositories.json")
+            .withStatus(200)));
+
+    stubFor(WireMock.get(urlEqualTo("/platform/platform-projects/_apis/git/repositories/my-helm-chart/items?path=.azureDevopsCrawler&"+API_VERSION))
+        .willReturn(aResponse().withStatus(404)));
+
+    stubFor(WireMock.get(urlEqualTo("/platform/platform-projects/_apis/git/repositories/vendor-portal-ui/items?path=.azureDevopsCrawler&"+API_VERSION))
+        .willReturn(aResponse().withStatus(404)));
+
+    stubFor(WireMock.get(urlEqualTo("/platform/platform-projects/_apis/git/repositories/my-helm-chart/items?path=pom.xml&versionDescriptor.versionType=branch&versionDescriptor.version=main&"+API_VERSION))
+        .willReturn((aResponse()
+            .withBody(FileUtils.readFileToString(ResourceUtils.getFile("classpath:sample_pom.xml"), "UTF-8"))
+            .withStatus(200))));
+
+    stubFor(WireMock.get(urlEqualTo("/platform/platform-projects/_apis/git/repositories/vendor-portal-ui/items?path=pom.xml&versionDescriptor.versionType=branch&versionDescriptor.version=main&"+API_VERSION))
+        .willReturn(aResponse().withStatus(404)));
+  }
+
+
+  @Test
+  void shouldCrawl() throws IOException {
+
+    var crawlers=context.getBeanNamesForType(GitHubCrawler.class);
+    log.info("found "+crawlers.length+" crawlers in context : "+crawlers[0]);
+
+    var crawlerFromContext= context.getBean(GitHubCrawler.class);
+    log.info("crawlerId : "+crawlerFromContext);
+    log.info("properties : "+crawlerFromContext.getGitHubCrawlerProperties());
+
+
+    var remoteGitHubs=context.getBeanNamesForType(RemoteGitHub.class);
+    log.info("found "+remoteGitHubs.length+" remoteGitHubs in context : "+remoteGitHubs[0]);
+    var remoteGitHubFromContext= context.getBean(RemoteGitHub.class);
+    log.info("remoteGitHubFromContextId : "+remoteGitHubFromContext);
+
+
+
+    log.info("about to start crawling...");
+
+    log.info("URL config used : "+crawler.getGitHubCrawlerProperties().getGithubConfig().getApiUrl());
+
+    log.info("nb misc tasks : "+crawler.getGitHubCrawlerProperties().getMiscRepositoryTasks().size());
+
+
+    try {
+      crawler.crawl();
+    } catch (IOException e) {
+      log.error("problem while crawling",e);
+      throw e;
+    }
+
+
+    //TODO have an in-memory output and assert content
+  }
+
+
+}

github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/GitHubCrawlerIT.java

@@ -26,13 +26,16 @@
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.test.annotation.DirtiesContext;
+import org.springframework.test.annotation.DirtiesContext.ClassMode;
 import org.springframework.test.context.ActiveProfiles;
 import org.springframework.test.context.junit.jupiter.SpringExtension;
 
 
 @ExtendWith(SpringExtension.class)
 @SpringBootTest(classes = {TestConfig.class, GitHubCrawlerAutoConfiguration.class})
-@ActiveProfiles(profiles = {"test", "profilesAreAWayOfGrouping"})
+@ActiveProfiles(profiles = {"gitHubTest", "profilesAreAWayOfGrouping"})
+@DirtiesContext(classMode = ClassMode.BEFORE_CLASS)
 public class GitHubCrawlerIT {
 
   private final int MAX_TIMEOUT_FOR_CRAWLER = 10;
@@ -75,11 +78,17 @@ public void mockSetUp() {
       hasGitHubMockServerStarted = true;
     }
 
-    output.reset();
     githubMockServer.reset();
+  }
+
+  @BeforeEach
+  void resetSharedData() {
+    output.reset();
+
     crawler.getGitHubCrawlerProperties().setRepositoriesToExclude(new ArrayList<>());
     crawler.getGitHubCrawlerProperties().setPublishExcludedRepositories(true);
     crawler.getGitHubCrawlerProperties().setCrawlAllBranches(false);
+    crawler.getTasksToPerform().clear();
   }
 
 
@@ -104,7 +113,7 @@ void shouldHitOrgaNextPageOfRepositoriesIfMorethanOne() throws IOException {
 
     assertThat(githubMockServer.isHasCalledNextPage()).as("next page wasn't called").isTrue();
 
-    assertThat(githubMockServer.getRepoConfigHits().size()).isGreaterThanOrEqualTo(nbRepositoriesInOrga);
+    assertThat(githubMockServer.getRepoConfigHits()).hasSizeGreaterThanOrEqualTo(nbRepositoriesInOrga);
     assertThat(githubMockServer.getNbPages()).isEqualTo(2);
 
   }
@@ -113,7 +122,7 @@ void shouldHitOrgaNextPageOfRepositoriesIfMorethanOne() throws IOException {
   void excludingRepositoriesOnServerConfigSideWithSingleRegexp() throws IOException {
 
     String excludedRepoName = "api-.*";
-    crawler.getGitHubCrawlerProperties().setRepositoriesToExclude(Arrays.asList(excludedRepoName));
+    crawler.getGitHubCrawlerProperties().setRepositoriesToExclude(List.of(excludedRepoName));
     crawler.crawl();
 
     assertOnlyThisRepoIsFlaggedAsExcluded("api-gateway");
@@ -162,7 +171,7 @@ void shouldCopyActiveProfilesAsGroupsOnRepo() throws IOException {
         .until(() -> assertThat(processedRepositories).hasSize(nbRepositoriesInOrga));
 
     processedRepositories.stream().forEach(repo -> {
-      assertThat(repo.getGroups()).containsExactlyInAnyOrder("test", "profilesAreAWayOfGrouping");
+      assertThat(repo.getGroups()).containsExactlyInAnyOrder("gitHubTest", "profilesAreAWayOfGrouping");
     });
 
   }
@@ -175,8 +184,9 @@ void shouldNotFetchFilesWithIndicatorsIfRepoIsExcluded() throws IOException {
 
     crawler.crawl();
 
-    assertThat(githubMockServer.getPomXmlHits()).hasSize(nbRepositoriesInOrga - 1);
-    assertThat(githubMockServer.getPomXmlHits()).doesNotContain(excludedRepoName);
+    assertThat(githubMockServer.getPomXmlHits())
+        .hasSize(nbRepositoriesInOrga - 1)
+        .doesNotContain(excludedRepoName);
   }
 
   @Test
@@ -305,6 +315,10 @@ void processNormallyIfNoConfigFileOnRepoSide() throws IOException {
   @Test
   void shouldPerformSearchOnAllRepos() throws IOException {
 
+    assertThat(output.getAnalyzedRepositories()).isEmpty();
+    assertThat(githubMockServer.getSearchHitsCount()).isEqualTo(0);
+
+
     crawler.crawl();
 
     Collection<Repository> actualRepositories = output.getAnalyzedRepositories().values();

github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/RemoteGitHubImplTest.kt

@@ -9,13 +9,15 @@ import org.junit.jupiter.api.BeforeAll
 import org.junit.jupiter.api.BeforeEach
 import org.junit.jupiter.api.Test
 import org.junit.jupiter.api.extension.ExtendWith
+import org.springframework.test.annotation.DirtiesContext
 
 import org.springframework.test.context.ActiveProfiles
 import org.springframework.test.context.junit.jupiter.SpringExtension
 import java.util.concurrent.TimeUnit.SECONDS
 
 @ExtendWith(SpringExtension::class)
-@ActiveProfiles(profiles = arrayOf("test"))
+@ActiveProfiles(profiles = arrayOf("gitHubTest"))
+@DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_CLASS)
 class RemoteGitHubImplTest {
 
     companion object {

github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/RemoteGitLabImplTest.kt

@@ -10,14 +10,16 @@ import org.junit.jupiter.api.BeforeAll
 import org.junit.jupiter.api.BeforeEach
 import org.junit.jupiter.api.Test
 import org.junit.jupiter.api.extension.ExtendWith
+import org.springframework.test.annotation.DirtiesContext
 
 import org.springframework.test.context.ActiveProfiles
 import org.springframework.test.context.junit.jupiter.SpringExtension
 import java.util.concurrent.TimeUnit.SECONDS
 
 
 @ExtendWith(SpringExtension::class)
-@ActiveProfiles(profiles = arrayOf("test"))
+@ActiveProfiles(profiles = arrayOf("gitLabTest"))
+@DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_CLASS)
 class RemoteGitLabImplTest {
 
     companion object {