Add more details status, including a todo page

config.ini

@@ -0,0 +1,9 @@
+[current]
+url=https://copr.fedorainfracloud.org
+owner=@fedora-llvm-team
+project=clang-built-f36
+
+[next]
+url=https://copr.fedorainfracloud.org
+owner=@fedora-llvm-team
+project=clang-built-f37

copr-reporter/config.ini

@@ -0,0 +1,7 @@
+[current]
+owner=@fedora-llvm-team
+project=clang-built-f35
+
+[next]
+owner=@fedora-llvm-team
+project=clang-built-f36

copr-reporter/f36.ini

@@ -0,0 +1,9 @@
+[current]
+url=https://copr.fedorainfracloud.org
+owner=@fedora-llvm-team
+project=clang-built-f35
+
+[next]
+url=https://copr.fedorainfracloud.org
+owner=@fedora-llvm-team
+project=clang-built-f36

copr-reporter/f37.ini

@@ -0,0 +1,9 @@
+[current]
+url=https://copr.fedorainfracloud.org
+owner=@fedora-llvm-team
+project=clang-built-f36
+
+[next]
+url=https://copr.fedorainfracloud.org
+owner=@fedora-llvm-team
+project=clang-built-f37

copr-reporter/html_generator.py

@@ -0,0 +1,64 @@
+import json
+import datetime
+import configparser
+import io
+import urllib.request
+
+import jinja2
+
+
+# TODO: Add a summary of the new broken packages
+# TODO: Add in the summary the total amount of packages
+def generate_report(title, results, description):
+    loader = jinja2.FileSystemLoader(searchpath="./")
+    environment = jinja2.Environment(loader=loader)
+    file = "template.html"
+
+    template = environment.get_template(file)
+    text = template.render(
+        title=title,
+        date=datetime.datetime.now().isoformat(),
+        results=results,
+        description = description
+    )
+    report = open("report.html", 'w')
+    report.write(text)
+    report.close()
+
+def get_combined_build_state(chroots):
+    state = 'succeeded'
+    for c in chroots:
+        chroot = chroots[c]
+        if chroot['state'] == 'failed':
+            state = 'failed'
+            break
+    return state
+
+if __name__ == '__main__':
+    title = "Clang Mass Rebuild TODO dashboard"
+    packages = {}
+    with open("packages.json", "r") as f:
+        packages = json.load(f)
+
+    current_ver = None
+    next_ver = None
+
+    for k, v in packages.items():
+        packages[k]['changed'] = 'Same results'
+        state_a = get_combined_build_state(packages[k]['builds_a']['chroots'])
+        state_b = get_combined_build_state(packages[k]['builds_b']['chroots'])
+        if state_a == 'succeeded' and state_b == 'failed':
+            packages[k]['changed'] = "Regression"
+        elif state_a == 'failed' and state_b == 'succeeded':
+            packages[k]['changed'] = "Fixed"
+        elif state_a != state_b:
+            packages[k]['changed'] = "Something has changed. You should verify the builds"
+
+        description = f"""
+    Explanation of the columns:<ul>
+    <li>Name: The name of the package.</li>
+    <li>Builds with current Clang release on {current_ver}.</li>
+    <li>Builds with next Clang release on {next_ver}.</li>
+    <li>Changes: Notes so you can search results easily.</li></ul>
+    """
+    generate_report(title, packages, description)

copr-reporter/json_generator.py

@@ -0,0 +1,106 @@
+import json
+import configparser
+from copr.v3 import Client
+from munch import Munch
+import sys
+
+
+def load_config(file):
+    c = configparser.ConfigParser()
+    c.read(file)
+    return c
+
+def create_missing_package(name):
+    return Munch(name = name, state = 'missing', builds = {'latest' : None })
+
+def create_missing_build(arch):
+    return Munch(name = arch, state = 'missing', result_url = '')
+
+def retrieve_packages(client, project_owner, project_name):
+    response = client.monitor_proxy.monitor(project_owner, project_name, additional_fields = ['url_build_log'])
+    return response['packages']
+
+def get_package(name, pkg_list):
+    for p in pkg_list:
+        if p['name'] == name:
+            return p
+
+def handle_missing_packages(packages_a, packages_b):
+
+    for p_a in packages_a:
+        p_b = get_package(p_a['name'], packages_b)
+        if not p_b:
+            packages_b.append({'name' : p_a['name'], 'chroots' : {} })
+
+    for p_b in packages_b:
+        p_a = get_package(p_b['name'], packages_a)
+        if not p_a:
+            packages_a.append({'name' : p_b['name'], 'chroots' : {} })
+
+def get_chroot_arch(chroot):
+    return chroot.split('-')[-1]
+
+def has_arch(builds, arch):
+    for build in builds:
+        if get_chroot_arch(build.name) == arch:
+            return True
+    return False
+
+def retrieve_builds(client, package):
+
+    latest_build = package.builds['latest']
+    if not latest_build:
+        return []
+
+    # When a build has state of succeeded, we don't need to query COPR for
+    # more information, because we know the builds have succeeded for all
+    # chroots.  A 'failed' state means that there was at least 1 failure,
+    # so in that case we still need to query COPR to see which chroots
+    # suceeded and which chroots failed.
+    if latest_build['state'] == 'succeeded':
+        builds = []
+        for c in sorted(latest_build['chroots']):
+            builds.append(Munch(name=c, result_url='{}/{}/0{}-{}/'.format(latest_build['repo_url'], c, latest_build['id'], package.name),
+                                           state = latest_build['state']))
+        return builds
+
+
+    b = client.build_chroot_proxy.get_list(package.builds['latest']['id'])
+    return b
+
+
+def create_copr_client(configfile = None, copr_url = None):
+    if copr_url:
+        config = {'copr_url' : copr_url }
+        return Client(config)
+
+    return Client.create_from_config_file(configfile)
+
+if __name__ == '__main__':
+    config_file = './config.ini'
+    if len(sys.argv) == 2:
+        config_file = sys.argv[1]
+    config = load_config(file=config_file)
+    client_a = create_copr_client(copr_url = config.get('current', 'url', fallback = None),
+                                  configfile = config.get('current', 'config', fallback = None))
+    client_b = create_copr_client(copr_url = config.get('next', 'url', fallback = None),
+                                  configfile = config.get('next', 'config', fallback = None))
+    packages_a = retrieve_packages(client_a, config['current']['owner'], config['current']['project'])
+    packages_b = retrieve_packages(client_b, config['next']['owner'], config['next']['project'])
+    handle_missing_packages(packages_a, packages_b)
+
+    project_b = client_b.project_proxy.get(config['next']['owner'], config['next']['project'])
+    chroot_b = list(project_b['chroot_repos'].keys())[0]
+    os_version_b = "-".join(chroot_b.split('-')[0:2])
+    packages = {}
+    sort_key = lambda x : x['name']
+    for pa, pb in zip(sorted(packages_a, key = sort_key), sorted(packages_b, key = sort_key)):
+        new_package = {'name' : pa['name'],
+                       'os_version' : os_version_b, 
+                       'builds_a' : pa,
+                       'builds_b' : pb
+                      }
+        packages[pa['name']] = new_package
+
+    with open("packages.json", "w") as out:
+        json.dump(packages, out, indent=2)

copr-reporter/template.html

@@ -0,0 +1,146 @@
+<!doctype html>
+<html lang="en">
+<head>
+    <meta charset="utf-8">
+    <meta http-equiv="X-UA-Compatible" content="IE=edge">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <meta name="description" content="Report of packages">
+    <meta name="author" content="alexsaezm">
+
+    <title>{{ title }}</title>
+    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap@5.1.3/dist/css/bootstrap.min.css"/>
+    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap-icons@1.7.2/font/bootstrap-icons.css"/>
+    <link rel="stylesheet" type="text/css" href="https://cdn.datatables.net/v/bs5/dt-1.11.3/datatables.min.css"/>
+    <style type="text/css" media="screen">
+        .importing, .pending {
+            color: #0dcaf0;
+        }
+
+        .starting, .running {
+            color: #ffc107;
+        }
+
+        .succeeded, .forked, .skipped {
+            color: #198754;
+        }
+
+        .failed {
+            color: #dc3545;
+        }
+
+        .canceled {
+            color: #212529;
+        }
+
+	.missing {
+            visibility: hidden;
+	}
+
+    </style>
+</head>
+<body>
+<div class="container">
+    <div class="page-header">
+        <h1>{{ title }}</h1>
+    <a href='f36-todo.html'>Fedora 36</a> <a href='f37-todo.html'>Fedora 37</a>
+    <a href='f37-status.html'>More Data</a><br><br>
+	{{ description }}
+        </p>
+    </div>
+    <div class="table-responsive">
+        <table id="packages" class="table table-striped" style="width:100%">
+            <thead>
+            <tr>
+                <th>Name</th>
+                <th>Builds with current clang release</th>
+                <th>Builds with next clang release</th>
+                <th>Changes</th>
+                <th>Notes</th>
+            </tr>
+            </thead>
+            <tfoot>
+            <tr>
+                <th>Name</th>
+                <th>Builds with current clang release</th>
+                <th>Builds with next clang release</th>
+                <th>Changes</th>
+                <th>Notes</th>
+            </tr>
+            </tfoot>
+            <tbody>
+            {% for name, data in results.items() -%}
+            <tr>
+                <td>{{ name }}</td>
+                <td>
+                    {% for k, b in data["builds_a"]["chroots"].items()|sort() %}
+                    <p class="{{ b.state }}">
+                        {% if b.state == "importing" or b.state == "pending" %}
+                        <i class="bi bi-hourglass-top"></i>
+                        {% endif %}
+                        {% if b.state == "starting" or b.state == "running" %}
+                        <i class="bi bi-hourglass-split"></i>
+                        {% endif %}
+                        {% if b.state == "succeeded" or b.state == "forked" or b.state == "skipped" %}
+                        <i class="bi bi-check-circle"></i>
+                        {% endif %}
+                        {% if b.state == "failed" %}
+                        <i class="bi bi-exclamation-octagon"></i>
+                        {% endif %}
+                        {% if b.state == "canceled" %}
+                        <i class="bi bi-slash-circle"></i>
+                        {% endif %}
+                        <a href="{{ b['url_build_log'] }}"> {{ k.split('-')[-1] }} </a>
+                    </p>
+                    {% endfor %}
+                </td>
+                <td>
+                    {% for k, b in data["builds_b"]["chroots"].items()|sort() %}
+                    <p class="{{ b.state }}">
+                        {% if b.state == "importing" or b.state == "pending" %}
+                        <i class="bi bi-hourglass-top"></i>
+                        {% endif %}
+                        {% if b.state == "starting" or b.state == "running" %}
+                        <i class="bi bi-hourglass-split"></i>
+                        {% endif %}
+                        {% if b.state == "succeeded" or b.state == "forked" or b.state == "skipped" %}
+                        <i class="bi bi-check-circle"></i>
+                        {% endif %}
+                        {% if b.state == "failed" %}
+                        <i class="bi bi-exclamation-octagon"></i>
+                        {% endif %}
+                        {% if b.state == "canceled" %}
+                        <i class="bi bi-slash-circle"></i>
+                        {% endif %}
+                        {% if b.url is not none %}
+                        <a href="{{ b.url_build_log }}"> {{ k.split('-')[-1] }} </a>
+                        {% else %}
+                        {{ b.arch }}
+                        {% endif %}
+
+                    </p>
+                    {% endfor %}
+                </td><td>
+                    {{ data['changed'] }}
+                </td>
+		<td>{{ data['note'] }}</td>
+            </tr>
+            {% endfor %}
+            </tbody>
+        </table>
+    </div>
+</div>
+<footer class="py-3 my-4">
+    <p class="text-center text-muted">Generated: {{ date }}</p>
+</footer>
+<script type="text/javascript" src="https://code.jquery.com/jquery-1.12.4.min.js"></script>
+<script src="https://cdn.jsdelivr.net/npm/bootstrap@5.1.3/dist/js/bootstrap.bundle.min.js"
+        integrity="sha384-ka7Sk0Gln4gmtz2MlQnikT1wXgYsOg+OMhuP+IlRH9sENBO0LRn5q+8nbTov4+1p"
+        crossorigin="anonymous"></script>
+<script type="text/javascript" src="https://cdn.datatables.net/v/bs5/dt-1.11.3/datatables.min.js"></script>
+<script>
+    $(document).ready(function () {
+        $('#packages').DataTable();
+    });
+</script>
+</body>
+</html>

f36.ini

@@ -0,0 +1,9 @@
+[current]
+url=https://copr.fedorainfracloud.org
+owner=@fedora-llvm-team
+project=clang-built-f35
+
+[next]
+url=https://copr.fedorainfracloud.org
+owner=@fedora-llvm-team
+project=clang-built-f36

f37.ini

@@ -0,0 +1,9 @@
+[current]
+url=https://copr.fedorainfracloud.org
+owner=@fedora-llvm-team
+project=clang-built-f36
+
+[next]
+url=https://copr.fedorainfracloud.org
+owner=@fedora-llvm-team
+project=clang-built-f37