Add gerrit manual repos and other fixes

Signed-off-by: Steve Winslow <swinslow@gmail.com>

gerrit.py

@@ -39,3 +39,8 @@ def getGerritRepoDict(apiurl):
     repos = parseRepoJSONData(rj)
     repodict = splitReposToDict(repos['active'])
     return dict(repodict)
+
+def getGerritRepoList(apiurl):
+    rj = getRepoJSONData(apiurl)
+    repos = parseRepoJSONData(rj)
+    return repos['active']

github.py

@@ -5,15 +5,15 @@
 
 GITHUB_API_URL = "https://api.github.com"
 
-def getOrgJSONData(org):
-    url = f"{GITHUB_API_URL}/orgs/{org}/repos?per_page=100"
+def getOrgJSONData(org, page):
+    url = f"{GITHUB_API_URL}/orgs/{org}/repos?page={page}&per_page=100"
     r = requests.get(url)
     if r.status_code == 200:
         return r.json()
 
     # if r is 404, then this might be a user, not an org -- try that instead
     if r.status_code == 404:
-        user_url = f"{GITHUB_API_URL}/users/{org}/repos?per_page=100"
+        user_url = f"{GITHUB_API_URL}/users/{org}/repos?{page}&per_page=100"
         r2 = requests.get(user_url)
         if r2.status_code == 200:
             return r2.json()
@@ -34,5 +34,18 @@ def parseOrgJSONData(rj):
     return repos
 
 def getGithubRepoList(org):
-    rj = getOrgJSONData(org)
-    return parseOrgJSONData(rj)
+    repos = []
+    stillSomeRepos = True
+    page = 1
+    while stillSomeRepos:
+        rj = getOrgJSONData(org, page)
+        gotRepos = parseOrgJSONData(rj)
+        if len(gotRepos) <= 0:
+            stillSomeRepos = False
+            break
+        else:
+            for r in gotRepos:
+                repos.append(r)
+            page += 1
+
+    return repos

repolisting.py

@@ -3,7 +3,7 @@
 
 from datatypes import ProjectRepoType, Status, Subproject
 from github import getGithubRepoList
-from gerrit import getGerritRepoDict
+from gerrit import getGerritRepoDict, getGerritRepoList
 
 # Runner for START in GitHub
 def doRepoListingForSubproject(cfg, prj, sp):
@@ -92,12 +92,11 @@ def doRepoListingForProject(cfg, prj):
 # Runner for START in GERRIT
 def doRepoListingForGerritProject(cfg, prj):
     if prj._gerrit_subproject_config == "auto":
-        doRepoListingForGerritAutoProject(cfg, prj)
+        return doRepoListingForGerritAutoProject(cfg, prj)
     elif prj._gerrit_subproject_config == "one":
-        doRepoListingForGerritOneProject(cfg, prj)
+        return doRepoListingForGerritOneProject(cfg, prj)
     elif prj._gerrit_subproject_config == "manual":
-        print(f"{prj._name}: subproject-config value of 'manual' not yet implemented")
-        return False
+        return doRepoListingForGerritManualProject(cfg, prj)
     else:
         print(f"{prj._name}: invalid subproject-config value: {prj._gerrit_subproject_config}")
         return False
@@ -205,6 +204,47 @@ def doRepoListingForGerritOneProject(cfg, prj):
 
 # Runner for START in GERRIT where subproject-config is manual
 def doRepoListingForGerritManualProject(cfg, prj):
-    # not yet implemented; need to account for grouping vs. flat repo names
-    # could have same repo names in different groupings
-    return False
+    # collect all configured repos, and what subprojects they're in
+    allcfgrepos = {}
+    for sp_name, sp in prj._subprojects.items():
+        for r in sp._repos:
+            allcfgrepos[r] = sp_name
+
+    # collect all real repos currently on Gerrit
+    allrealrepos = getGerritRepoList(prj._gerrit_apiurl)
+
+    # first, figure out what repos need to be added
+    for r in allrealrepos:
+        config_sp = allcfgrepos.get(r, "")
+        if config_sp == "" and r not in prj._gerrit_repos_ignore and r not in prj._gerrit_repos_pending:
+            prj._gerrit_repos_pending.append(r)
+            print(f"{prj._name}: new pending repo: {r}")
+
+    # then, figure out what repos need to be removed
+    for sp_name, sp in prj._subprojects.items():
+        repos_to_remove = []
+        for r in sp._repos:
+            if r not in allrealrepos:
+                repos_to_remove.append(r)
+        for r in repos_to_remove:
+            sp._repos.remove(r)
+            print(f"{prj._name}/{sp._name}: removed {r} from repos")
+
+    repos_ignore_to_remove = []
+    for r in prj._gerrit_repos_ignore:
+        if r not in allrealrepos:
+            repos_ignore_to_remove.append(r)
+    for r in repos_ignore_to_remove:
+        prj._gerrit_repos_ignore.remove(r)
+        print(f"{prj._name}: removed {r} from repos-ignore")
+
+    # finally, throw a "fail" if any new repos are pending
+    if len(prj._gerrit_repos_pending) > 0:
+        print(f"{prj._name}: stopped, need to assign repos-pending")
+        return False
+    else:
+        # success - advance state
+        prj._status = Status.GOTLISTING
+        for _, sp in prj._subprojects.items():
+            sp._status = Status.GOTLISTING
+        return True

runagents.py

@@ -14,7 +14,7 @@ def priorUploadExists(fdServer, priorUploadFolder, priorUploadName):
     oldFolderNum = fdServer.GetFolderNum(priorUploadFolder)
     if oldFolderNum is None or oldFolderNum == -1:
         return False
-    oldUploadNum = fdServer.GetUploadNum(oldFolderNum, priorUploadName)
+    oldUploadNum = fdServer.GetUploadNum(oldFolderNum, priorUploadName, False)
     if oldUploadNum is None or oldUploadNum == -1:
         return False
     # if we get here, the old scan exists
@@ -54,6 +54,7 @@ def doRunAgentsForSubproject(cfg, fdServer, prj, sp):
     if priorUploadExists(fdServer, priorFolder, priorUploadFragment):
         print(f"{prj._name}/{sp._name}: running reuser")
         t = Reuse(fdServer, uploadName, uploadFolder, priorUploadFragment, priorFolder)
+        t.exact = False
         retval = t.run()
         if not retval:
             print(f"{prj._name}/{sp._name}: error running reuse from {priorUploadFragment} in {priorFolder}")

scaffold.py

@@ -32,6 +32,8 @@ def status(projects, prj_only, sp_only):
 
     for prj in projects.values():
         if prj_only == "" or prj_only == prj._name:
+            row = [prj._name, "", prj._status.name]
+            table.append(row)
             for sp in prj._subprojects.values():
                 if sp_only == "" or sp_only == sp._name:
                     row = [prj._name, sp._name, sp._status.name]