diff --git a/.gitignore b/.gitignore index 70b9777..2ffef55 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ keys/** __pycache__/ *.py[cod] *$py.class +*.bkup # C extensions *.so diff --git a/.vscode/launch.json b/.vscode/launch.json index 9eb5819..e307130 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -4,6 +4,7 @@ // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 "version": "0.2.0", "configurations": [ + { "name": "sheets-to-csv", "type": "python", @@ -27,6 +28,14 @@ "program": "tools/process-results.py", "console": "integratedTerminal", "justMyCode": true + }, + { + "name": "team-results", + "type": "python", + "request": "launch", + "program": "tools/team-results.py", + "console": "integratedTerminal", + "justMyCode": true } ] } \ No newline at end of file diff --git a/data/2022-23/2/club-parsed/34.U15_Combined.csv b/data/2022-23/2/club-parsed/34.U15_Combined.csv new file mode 100644 index 0000000..d8dbc75 --- /dev/null +++ b/data/2022-23/2/club-parsed/34.U15_Combined.csv @@ -0,0 +1,6 @@ +,names,gender +0,Rhuridh Miller,m +1,Lauren,f +2,Fraser McKillop,m +3,Lucas Watt,m +4,Alex Massie,m diff --git a/results/provisional/2022-23/1/html/index.html b/results/provisional/2022-23/1/html/index.html index b1c498f..41a693e 100644 --- a/results/provisional/2022-23/1/html/index.html +++ b/results/provisional/2022-23/1/html/index.html @@ -1,112 +1,112 @@ - +
- + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + +
 resultsPageresultsPage
0U11_Boys.U11.male.team.results.html0U11_Boys.U11.male.team.results.html
1U11_Boys.results.html1U11_Boys.results.html
2U11_Girls.U11.female.team.results.html2U11_Girls.U11.female.team.results.html
3U11_Girls.results.html3U11_Girls.results.html
4U13_Boys.U13.male.team.results.html4U13_Boys.U13.male.team.results.html
5U13_Boys.results.html5U13_Boys.results.html
6U13_Girls.U13.female.team.results.html6U13_Girls.U13.female.team.results.html
7U13_Girls.results.html7U13_Girls.results.html
8U15_Combined.U15.female.team.results.html8U15_Combined.U15.female.team.results.html
9U15_Combined.U15.male.team.results.html9U15_Combined.U15.male.team.results.html
10U15_Combined.results.html10U15_Combined.results.html
11U17_Combined.U17.female.team.results.html11U17_Combined.U17.female.team.results.html
12U17_Combined.U17.male.team.results.html12U17_Combined.U17.male.team.results.html
13U17_Combined.results.html13U17_Combined.results.html
14U20-Senior-Masters_Combined.MASTER.female.team.results.html14U20-Senior-Masters_Combined.MASTER.female.team.results.html
15U20-Senior-Masters_Combined.MASTER.male.team.results.html15U20-Senior-Masters_Combined.MASTER.male.team.results.html
16U20-Senior-Masters_Combined.OVERALL.female.team.results.html16U20-Senior-Masters_Combined.OVERALL.female.team.results.html
17U20-Senior-Masters_Combined.OVERALL.male.team.results.html17U20-Senior-Masters_Combined.OVERALL.male.team.results.html
18U20-Senior-Masters_Combined.SENIOR.female.team.results.html18U20-Senior-Masters_Combined.SENIOR.female.team.results.html
19U20-Senior-Masters_Combined.SENIOR.male.team.results.html19U20-Senior-Masters_Combined.SENIOR.male.team.results.html
20U20-Senior-Masters_Combined.U20.female.team.results.html20U20-Senior-Masters_Combined.U20.female.team.results.html
21U20-Senior-Masters_Combined.U20.male.team.results.html21U20-Senior-Masters_Combined.U20.male.team.results.html
22U20-Senior-Masters_Combined.results.html22U20-Senior-Masters_Combined.results.html
23missingTeamSubmissions.html23missingTeamSubmissions.html
24volunteers.html24volunteers.html
diff --git a/results/provisional/2022-23/1/meta.json b/results/provisional/2022-23/1/meta.json new file mode 100644 index 0000000..a7d4927 --- /dev/null +++ b/results/provisional/2022-23/1/meta.json @@ -0,0 +1,101 @@ +{ + "races": { + "U13": { + "M": { + "penalty": 38, + "participants": 28, + "counters": 3 + }, + "F": { + "penalty": 62, + "participants": 52, + "counters": 3 + } + }, + "U11": { + "M": { + "penalty": 49, + "participants": 39, + "counters": 3 + }, + "F": { + "penalty": 68, + "participants": 58, + "counters": 3 + } + }, + "U15": { + "M": { + "penalty": 42, + "participants": 32, + "counters": 3 + }, + "F": { + "penalty": 40, + "participants": 30, + "counters": 3 + } + }, + "U17": { + "M": { + "penalty": 27, + "participants": 17, + "counters": 3 + }, + "F": { + "penalty": 27, + "participants": 17, + "counters": 3 + } + }, + "U20": { + "M": { + "penalty": 12, + "participants": 2, + "counters": 3 + }, + "F": { + "penalty": 12, + "participants": 2, + "counters": 3 + } + }, + "SENIOR": { + "M": { + "penalty": 67, + "participants": 57, + "counters": 4 + }, + "F": { + "penalty": 29, + "participants": 19, + "counters": 4 + } + }, + "MASTER": { + "M": { + "penalty": 70, + "participants": 60, + "counters": 3 + }, + "F": { + "penalty": 31, + "participants": 21, + "counters": 3 + } + }, + "OVERALL": { + "M": { + "penalty": 129, + "participants": 119, + "counters": 4 + }, + "F": { + "penalty": 52, + "participants": 42, + "counters": 4 + } + } + }, + "attendance": 434 +} \ No newline at end of file diff --git a/results/provisional/2022-23/2/.DS_Store b/results/provisional/2022-23/2/.DS_Store deleted file mode 100644 index 9166a8d..0000000 Binary files a/results/provisional/2022-23/2/.DS_Store and /dev/null differ diff --git a/results/provisional/2022-23/2/U15_Combined.results.csv b/results/provisional/2022-23/2/U15_Combined.results.csv index 64c4c89..a540398 100644 --- a/results/provisional/2022-23/2/U15_Combined.results.csv +++ b/results/provisional/2022-23/2/U15_Combined.results.csv @@ -13,16 +13,16 @@ position,times,team,Name,gender,AgeCat,clubnumber,Club name,Website 12,0:14:08,1M,BEN BAILLIE,M,U15,1,East Kilbride AC,http://www.ekac.org.uk/ 13,0:14:13,14M,Gregor Samson,M,U15,14,Ayr Seaforth AC,https://www.ayrseaforth.co.uk/ 14,0:14:28,2M,Lewis Anderson,M,U15,2,Kilmarnock H&AC,http://www.kilmarnockharriers.com/ -15,0:14:36,34M,,M,U15,34,Kilbarchan AAC,https://kilbarchanaac.org.uk/ +15,0:14:36,34M,Rhuridh Miller,M,U15,34,Kilbarchan AAC,https://kilbarchanaac.org.uk/ 16,0:14:42,6M,Euan Reid,M,U15,6,Cambuslang Harriers,https://cambuslangharriers.org/ 17,0:14:48,10F,Katie Woods,F,U15,10,Shettleston Harriers,http://shettlestonharriers.org.uk/ 18,0:14:57,7M,Dillon Sim,M,U15,7,Giffnock North AC,https://www.giffnocknorth.co.uk/ 19,0:15:24,1M,EUAN THORPE,M,U15,1,East Kilbride AC,http://www.ekac.org.uk/ 20,0:15:40,37M,Daniel Simpson,M,U15,37,Law & District AAC,http://www.lawaac.co.uk/ -21,0:15:43,34F,,F,U15,34,Kilbarchan AAC,https://kilbarchanaac.org.uk/ +21,0:15:43,34F,Lauren,F,U15,34,Kilbarchan AAC,https://kilbarchanaac.org.uk/ 22,0:15:46,11M,Murryn Burns,M,U15,11,Airdrie Harriers,http://airdrieharriers.org/ 23,0:15:54,14F,Marnie Harrower,F,U15,14,Ayr Seaforth AC,https://www.ayrseaforth.co.uk/ -24,0:15:59,34M,,M,U15,34,Kilbarchan AAC,https://kilbarchanaac.org.uk/ +24,0:15:59,34M,Fraser McKillop,M,U15,34,Kilbarchan AAC,https://kilbarchanaac.org.uk/ 25,0:16:00,36M,AARON SCOTT,M,U15,36,Larkhall YMCA,https://www.facebook.com/larkhallharriers/ 26,0:16:01,1F,EVA MORRISON,F,U15,1,East Kilbride AC,http://www.ekac.org.uk/ 27,0:16:14,7F,Isla Munro,F,U15,7,Giffnock North AC,https://www.giffnocknorth.co.uk/ @@ -35,8 +35,8 @@ position,times,team,Name,gender,AgeCat,clubnumber,Club name,Website 34,0:17:21,1F,CARRIE CHARTERS,F,U15,1,East Kilbride AC,http://www.ekac.org.uk/ 35,0:17:54,1M,CHRISTOPHER PATON,M,U15,1,East Kilbride AC,http://www.ekac.org.uk/ 36,0:17:59,10F,Katie Burns,F,U15,10,Shettleston Harriers,http://shettlestonharriers.org.uk/ -37,0:18:03,34M,,M,U15,34,Kilbarchan AAC,https://kilbarchanaac.org.uk/ -38,0:18:35,34M,,M,U15,34,Kilbarchan AAC,https://kilbarchanaac.org.uk/ +37,0:18:03,34M,Lucas Watt,M,U15,34,Kilbarchan AAC,https://kilbarchanaac.org.uk/ +38,0:18:35,34M,Alex Massie,M,U15,34,Kilbarchan AAC,https://kilbarchanaac.org.uk/ 39,0:19:29,2F,Charlotte Gebbie,F,U15,2,Kilmarnock H&AC,http://www.kilmarnockharriers.com/ 40,0:19:32,2F,Isla Fitzgerald,F,U15,2,Kilmarnock H&AC,http://www.kilmarnockharriers.com/ 41,0:20:15,7F,Lexie Martin,F,U15,7,Giffnock North AC,https://www.giffnocknorth.co.uk/ diff --git a/results/provisional/2022-23/2/html/U15_Combined.results.html b/results/provisional/2022-23/2/html/U15_Combined.results.html index 5f1a1f5..8ca55c6 100644 --- a/results/provisional/2022-23/2/html/U15_Combined.results.html +++ b/results/provisional/2022-23/2/html/U15_Combined.results.html @@ -171,7 +171,7 @@ 15 0:14:36 34M - + Rhuridh Miller M U15 34 @@ -237,7 +237,7 @@ 21 0:15:43 34F - + Lauren F U15 34 @@ -270,7 +270,7 @@ 24 0:15:59 34M - + Fraser McKillop M U15 34 @@ -413,7 +413,7 @@ 37 0:18:03 34M - + Lucas Watt M U15 34 @@ -424,7 +424,7 @@ 38 0:18:35 34M - + Alex Massie M U15 34 diff --git a/results/provisional/2022-23/2/html/index.html b/results/provisional/2022-23/2/html/index.html index 277e982..887abbb 100644 --- a/results/provisional/2022-23/2/html/index.html +++ b/results/provisional/2022-23/2/html/index.html @@ -1,112 +1,112 @@ - +
- + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + +
 resultsPageresultsPage
0U11_Boys.U11.male.team.results.html0U11_Boys.U11.male.team.results.html
1U11_Boys.results.html1U11_Boys.results.html
2U11_Girls.U11.female.team.results.html2U11_Girls.U11.female.team.results.html
3U11_Girls.results.html3U11_Girls.results.html
4U13_Boys.U13.male.team.results.html4U13_Boys.U13.male.team.results.html
5U13_Boys.results.html5U13_Boys.results.html
6U13_Girls.U13.female.team.results.html6U13_Girls.U13.female.team.results.html
7U13_Girls.results.html7U13_Girls.results.html
8U15_Combined.U15.female.team.results.html8U15_Combined.U15.female.team.results.html
9U15_Combined.U15.male.team.results.html9U15_Combined.U15.male.team.results.html
10U15_Combined.results.html10U15_Combined.results.html
11U17_Combined.U17.female.team.results.html11U17_Combined.U17.female.team.results.html
12U17_Combined.U17.male.team.results.html12U17_Combined.U17.male.team.results.html
13U17_Combined.results.html13U17_Combined.results.html
14U20-Senior-Masters_Combined.MASTER.female.team.results.html14U20-Senior-Masters_Combined.MASTER.female.team.results.html
15U20-Senior-Masters_Combined.MASTER.male.team.results.html15U20-Senior-Masters_Combined.MASTER.male.team.results.html
16U20-Senior-Masters_Combined.OVERALL.female.team.results.html16U20-Senior-Masters_Combined.OVERALL.female.team.results.html
17U20-Senior-Masters_Combined.OVERALL.male.team.results.html17U20-Senior-Masters_Combined.OVERALL.male.team.results.html
18U20-Senior-Masters_Combined.SENIOR.female.team.results.html18U20-Senior-Masters_Combined.SENIOR.female.team.results.html
19U20-Senior-Masters_Combined.SENIOR.male.team.results.html19U20-Senior-Masters_Combined.SENIOR.male.team.results.html
20U20-Senior-Masters_Combined.U20.female.team.results.html20U20-Senior-Masters_Combined.U20.female.team.results.html
21U20-Senior-Masters_Combined.U20.male.team.results.html21U20-Senior-Masters_Combined.U20.male.team.results.html
22U20-Senior-Masters_Combined.results.html22U20-Senior-Masters_Combined.results.html
23missingTeamSubmissions.html23missingTeamSubmissions.html
24volunteers.html24volunteers.html
diff --git a/results/provisional/2022-23/2/markdown/U15_Combined.results.md b/results/provisional/2022-23/2/markdown/U15_Combined.results.md index 556dbad..0ea19c0 100644 --- a/results/provisional/2022-23/2/markdown/U15_Combined.results.md +++ b/results/provisional/2022-23/2/markdown/U15_Combined.results.md @@ -14,16 +14,16 @@ | 12 | 0:14:08 | 1M | BEN BAILLIE | M | U15 | 1 | East Kilbride AC | http://www.ekac.org.uk/ | | 13 | 0:14:13 | 14M | Gregor Samson | M | U15 | 14 | Ayr Seaforth AC | https://www.ayrseaforth.co.uk/ | | 14 | 0:14:28 | 2M | Lewis Anderson | M | U15 | 2 | Kilmarnock H&AC | http://www.kilmarnockharriers.com/ | -| 15 | 0:14:36 | 34M | nan | M | U15 | 34 | Kilbarchan AAC | https://kilbarchanaac.org.uk/ | +| 15 | 0:14:36 | 34M | Rhuridh Miller | M | U15 | 34 | Kilbarchan AAC | https://kilbarchanaac.org.uk/ | | 16 | 0:14:42 | 6M | Euan Reid | M | U15 | 6 | Cambuslang Harriers | https://cambuslangharriers.org/ | | 17 | 0:14:48 | 10F | Katie Woods | F | U15 | 10 | Shettleston Harriers | http://shettlestonharriers.org.uk/ | | 18 | 0:14:57 | 7M | Dillon Sim | M | U15 | 7 | Giffnock North AC | https://www.giffnocknorth.co.uk/ | | 19 | 0:15:24 | 1M | EUAN THORPE | M | U15 | 1 | East Kilbride AC | http://www.ekac.org.uk/ | | 20 | 0:15:40 | 37M | Daniel Simpson | M | U15 | 37 | Law & District AAC | http://www.lawaac.co.uk/ | -| 21 | 0:15:43 | 34F | nan | F | U15 | 34 | Kilbarchan AAC | https://kilbarchanaac.org.uk/ | +| 21 | 0:15:43 | 34F | Lauren | F | U15 | 34 | Kilbarchan AAC | https://kilbarchanaac.org.uk/ | | 22 | 0:15:46 | 11M | Murryn Burns | M | U15 | 11 | Airdrie Harriers | http://airdrieharriers.org/ | | 23 | 0:15:54 | 14F | Marnie Harrower | F | U15 | 14 | Ayr Seaforth AC | https://www.ayrseaforth.co.uk/ | -| 24 | 0:15:59 | 34M | nan | M | U15 | 34 | Kilbarchan AAC | https://kilbarchanaac.org.uk/ | +| 24 | 0:15:59 | 34M | Fraser McKillop | M | U15 | 34 | Kilbarchan AAC | https://kilbarchanaac.org.uk/ | | 25 | 0:16:00 | 36M | AARON SCOTT | M | U15 | 36 | Larkhall YMCA | https://www.facebook.com/larkhallharriers/ | | 26 | 0:16:01 | 1F | EVA MORRISON | F | U15 | 1 | East Kilbride AC | http://www.ekac.org.uk/ | | 27 | 0:16:14 | 7F | Isla Munro | F | U15 | 7 | Giffnock North AC | https://www.giffnocknorth.co.uk/ | @@ -36,8 +36,8 @@ | 34 | 0:17:21 | 1F | CARRIE CHARTERS | F | U15 | 1 | East Kilbride AC | http://www.ekac.org.uk/ | | 35 | 0:17:54 | 1M | CHRISTOPHER PATON | M | U15 | 1 | East Kilbride AC | http://www.ekac.org.uk/ | | 36 | 0:17:59 | 10F | Katie Burns | F | U15 | 10 | Shettleston Harriers | http://shettlestonharriers.org.uk/ | -| 37 | 0:18:03 | 34M | nan | M | U15 | 34 | Kilbarchan AAC | https://kilbarchanaac.org.uk/ | -| 38 | 0:18:35 | 34M | nan | M | U15 | 34 | Kilbarchan AAC | https://kilbarchanaac.org.uk/ | +| 37 | 0:18:03 | 34M | Lucas Watt | M | U15 | 34 | Kilbarchan AAC | https://kilbarchanaac.org.uk/ | +| 38 | 0:18:35 | 34M | Alex Massie | M | U15 | 34 | Kilbarchan AAC | https://kilbarchanaac.org.uk/ | | 39 | 0:19:29 | 2F | Charlotte Gebbie | F | U15 | 2 | Kilmarnock H&AC | http://www.kilmarnockharriers.com/ | | 40 | 0:19:32 | 2F | Isla Fitzgerald | F | U15 | 2 | Kilmarnock H&AC | http://www.kilmarnockharriers.com/ | | 41 | 0:20:15 | 7F | Lexie Martin | F | U15 | 7 | Giffnock North AC | https://www.giffnocknorth.co.uk/ | diff --git a/results/provisional/2022-23/2/meta.json b/results/provisional/2022-23/2/meta.json new file mode 100644 index 0000000..1825ee9 --- /dev/null +++ b/results/provisional/2022-23/2/meta.json @@ -0,0 +1,101 @@ +{ + "races": { + "U13": { + "M": { + "penalty": 34, + "participants": 24, + "counters": 3 + }, + "F": { + "penalty": 46, + "participants": 36, + "counters": 3 + } + }, + "U11": { + "M": { + "penalty": 47, + "participants": 37, + "counters": 3 + }, + "F": { + "penalty": 46, + "participants": 36, + "counters": 3 + } + }, + "U15": { + "M": { + "penalty": 37, + "participants": 27, + "counters": 3 + }, + "F": { + "penalty": 25, + "participants": 15, + "counters": 3 + } + }, + "U17": { + "M": { + "penalty": 25, + "participants": 15, + "counters": 3 + }, + "F": { + "penalty": 18, + "participants": 8, + "counters": 3 + } + }, + "U20": { + "M": { + "penalty": 14, + "participants": 4, + "counters": 3 + }, + "F": { + "penalty": 11, + "participants": 1, + "counters": 3 + } + }, + "SENIOR": { + "M": { + "penalty": 45, + "participants": 35, + "counters": 4 + }, + "F": { + "penalty": 29, + "participants": 19, + "counters": 4 + } + }, + "MASTER": { + "M": { + "penalty": 65, + "participants": 55, + "counters": 3 + }, + "F": { + "penalty": 24, + "participants": 14, + "counters": 3 + } + }, + "OVERALL": { + "M": { + "penalty": 104, + "participants": 94, + "counters": 4 + }, + "F": { + "penalty": 44, + "participants": 34, + "counters": 4 + } + } + }, + "attendance": 326 +} \ No newline at end of file diff --git a/tools/process-results.py b/tools/process-results.py index 634bb3f..ea52ec7 100755 --- a/tools/process-results.py +++ b/tools/process-results.py @@ -18,16 +18,22 @@ HTML_DIR, MARKDOWN_DIR, YEAR, - EVENT + EVENT, ) from src.utils_functions import fetch_events_from_dir, fetch_volunteers_from_dir from src.adapter_gender import gender_process from src.adapter_team import process_teams, load_team_submissions -from src.adapter_results import results_merge, tidy_results, merge_runners, get_missing_teams +from src.adapter_results import ( + results_merge, + tidy_results, + merge_runners, + get_missing_teams, +) from src.adapter_times import adjust_times from src.adapter_places import adjust_places, process_final_results from src.adapter_points import calculate_competition_points from src.adapter_pretty_html import render +from src.adapter_json import json_write import pandas as pd import pathlib from pretty_html_table import build_table @@ -43,8 +49,8 @@ SOURCE_DATA = DATA_DIR pathlib.Path(RESULTS_DIR).mkdir(parents=True, exist_ok=True) -pathlib.Path(RESULTS_DIR+MARKDOWN_DIR).mkdir(parents=True, exist_ok=True) -pathlib.Path(RESULTS_DIR+HTML_DIR).mkdir(parents=True, exist_ok=True) +pathlib.Path(RESULTS_DIR + MARKDOWN_DIR).mkdir(parents=True, exist_ok=True) +pathlib.Path(RESULTS_DIR + HTML_DIR).mkdir(parents=True, exist_ok=True) teams = pd.read_csv(TEAMS, index_col="Number") @@ -54,7 +60,9 @@ leagueResults = {} teamResults = {} missingTeams = set() +eventMeta = {} index = [] +eventTotalParticipants = 0 # for each event we have files for for event in fetch_events_from_dir(DATA_DIR): logging.debug(f"Processing: {event}") @@ -104,8 +112,7 @@ results[event] = process_final_results( tidy_results( merge_runners( - results_merge( - times=record["times"], places=record["places"]), + results_merge(times=record["times"], places=record["places"]), clubSubmissions=clubSubmissions, event=event, ) @@ -115,11 +122,12 @@ except Exception as e: raise Exception(f"Problem processing event {event}: {e}") - missingTeams = missingTeams.union(set(get_missing_teams( - results=results[event], submissions=clubSubmissions))) + missingTeams = missingTeams.union( + set(get_missing_teams(results=results[event], submissions=clubSubmissions)) + ) # Now we have a results DataFrame, so process the competition results - teamResults[event] = calculate_competition_points( + (racemeta, teamResults[event]) = calculate_competition_points( results=results[event], teams=teams, event=event ) @@ -127,36 +135,59 @@ # core results if results[event] is not None: - results[event].to_csv(RESULTS_DIR + "/" + event + - ".results.csv", index=False) + results[event].to_csv(RESULTS_DIR + "/" + event + ".results.csv", index=False) results[event].to_markdown( - RESULTS_DIR+MARKDOWN_DIR + "/" + event + ".results.md", index=False) - render(df=results[event], style='blue_light', - filename=RESULTS_DIR+HTML_DIR + "/" + event + ".results.html") + RESULTS_DIR + MARKDOWN_DIR + "/" + event + ".results.md", index=False + ) + render( + df=results[event], + style="blue_light", + filename=RESULTS_DIR + HTML_DIR + "/" + event + ".results.html", + ) index.append(event + ".results.html") else: raise Exception(f"Unexpectedly no merged results for event {event}") + eventTotalParticipants += racemeta["total_participants"] + # team results if teamResults[event] is not None: for gender in teamResults[event]: for competition in teamResults[event][gender]: - baseFileName = event + "." + competition + \ - "." + GENDER_COMPETITION_MAP[gender] + + if competition not in eventMeta: + eventMeta[competition] = {} + + eventMeta[competition][gender] = racemeta[competition][gender] + + baseFileName = ( + event + "." + competition + "." + GENDER_COMPETITION_MAP[gender] + ) teamResults[event][gender][competition].to_csv( - RESULTS_DIR + "/"+baseFileName + ".team.results.csv", + RESULTS_DIR + "/" + baseFileName + ".team.results.csv", index=False, ) teamResults[event][gender][competition].to_markdown( - RESULTS_DIR+MARKDOWN_DIR + "/" + baseFileName + ".team.results.md", + RESULTS_DIR + + MARKDOWN_DIR + + "/" + + baseFileName + + ".team.results.md", index=False, ) - render(df=teamResults[event][gender][competition], style='blue_light', filename=RESULTS_DIR+HTML_DIR+"/" - + baseFileName + ".team.results.html") + render( + df=teamResults[event][gender][competition], + style="blue_light", + filename=RESULTS_DIR + + HTML_DIR + + "/" + + baseFileName + + ".team.results.html", + ) index.append(baseFileName + ".team.results.html") - logging.info("Wrote: "+baseFileName + ".team.results.html") + logging.info("Wrote: " + baseFileName + ".team.results.html") else: raise Exception( f"Unexpectedly no merged results for team results in event {event}" @@ -164,46 +195,61 @@ # TODO: This all needs refactoring/tidying up +json_write( + object={"races": eventMeta, "attendance": eventTotalParticipants}, + filename=RESULTS_DIR + "/" + "meta.json", +) -missing = pd.DataFrame({'team': list(missingTeams)}) +missing = pd.DataFrame({"team": list(missingTeams)}) theMissingTeams = missing.join( - other=teams, on='team', lsuffix="missing", rsuffix="teamDetails") -theMissingTeams.to_csv(RESULTS_DIR + "/" + - "missingTeamSubmissions.csv", index=False) + other=teams, on="team", lsuffix="missing", rsuffix="teamDetails" +) +theMissingTeams.to_csv(RESULTS_DIR + "/" + "missingTeamSubmissions.csv", index=False) theMissingTeams.to_markdown( - RESULTS_DIR+MARKDOWN_DIR + "/" + "missingTeamSubmissions.md", index=False) -render(df=theMissingTeams, style='blue_light', filename=RESULTS_DIR + - HTML_DIR+"/"+"missingTeamSubmissions.html") + RESULTS_DIR + MARKDOWN_DIR + "/" + "missingTeamSubmissions.md", index=False +) +render( + df=theMissingTeams, + style="blue_light", + filename=RESULTS_DIR + HTML_DIR + "/" + "missingTeamSubmissions.html", +) index.append("missingTeamSubmissions.html") volunteersFile = fetch_volunteers_from_dir(dir=DATA_DIR) if volunteersFile: - volunteers = pd.read_csv( - volunteersFile - ) - volunteers.columns = ['Name', 'Role'] + volunteers = pd.read_csv(volunteersFile) + volunteers.columns = ["Name", "Role"] volunteers.to_csv(RESULTS_DIR + "/" + "volunteers.csv", index=False) volunteers.to_markdown( - RESULTS_DIR+MARKDOWN_DIR + "/" + "volunteers.md", index=False) - render(df=volunteers, style='blue_light', - filename=RESULTS_DIR+HTML_DIR+"/"+"volunteers.html") + RESULTS_DIR + MARKDOWN_DIR + "/" + "volunteers.md", index=False + ) + render( + df=volunteers, + style="blue_light", + filename=RESULTS_DIR + HTML_DIR + "/" + "volunteers.html", + ) index.append("volunteers.html") # Render a basic HTML index + def make_clickable(val): return f'{val}' index.sort() -indexDF = pd.DataFrame({'resultsPage': index}) -indexDF = indexDF.style.format({'resultsPage': make_clickable}) - -indexDF.to_html(RESULTS_DIR+HTML_DIR+"/"+"index.html", - index=False, render_links=True, escape=False) +indexDF = pd.DataFrame({"resultsPage": index}) +indexDF = indexDF.style.format({"resultsPage": make_clickable}) + +indexDF.to_html( + RESULTS_DIR + HTML_DIR + "/" + "index.html", + index=False, + render_links=True, + escape=False, +) logging.info("Finished") diff --git a/tools/src/adapter_format.py b/tools/src/adapter_format.py new file mode 100644 index 0000000..5e4fdd3 --- /dev/null +++ b/tools/src/adapter_format.py @@ -0,0 +1,15 @@ +import pandas as pd +from .utils_consts import RESULTS_DIR, MARKDOWN_DIR, HTML_DIR +from .adapter_pretty_html import render + + +def export_results(results=None, results_dir: str = RESULTS_DIR, base_file_name: str = None, suffix: str = None, index: bool = False): + + files = {'csv': results_dir+"/"+base_file_name+suffix+".csv", + 'markdown': results_dir+"/"+MARKDOWN_DIR+"/"+base_file_name+suffix+".md", + 'html': results_dir+HTML_DIR+"/"+base_file_name + suffix+".html"} + results.to_csv(files['csv'], index=index) + results.to_markdown(files['markdown'], index=index) + render(df=results, style='blue_light', filename=files['html']) + + return files diff --git a/tools/src/adapter_json.py b/tools/src/adapter_json.py new file mode 100644 index 0000000..b241c83 --- /dev/null +++ b/tools/src/adapter_json.py @@ -0,0 +1,13 @@ +import json + + +def json_write(object=None, filename: str = None): + + with open(filename, "w") as fh: + json.dump(object, fh, indent=6) + + +def json_load(filename: str = None): + with open(filename, "r") as fh: + object = json.load(fh) + return object diff --git a/tools/src/adapter_points.py b/tools/src/adapter_points.py index e7c525b..53b7a60 100644 --- a/tools/src/adapter_points.py +++ b/tools/src/adapter_points.py @@ -68,7 +68,8 @@ def extract_filtered_results(results=None, ageCat=None, gender=None): # extract the matching gender records if gender is not None: - genderResults = ageCatResults[ageCatResults["gender"] == gender].reset_index() + genderResults = ageCatResults[ageCatResults["gender"] + == gender].reset_index() else: genderResults = ageCatResults.reset_index() @@ -89,7 +90,8 @@ def calculate_team_points(team, teamResults, maxCounters, penaltyPoints): # what were the finisher positions teamFinishers = ",".join( - teamResults.head(maxCounters)["position"].astype("string").values.tolist() + teamResults.head(maxCounters)["position"].astype( + "string").values.tolist() ) totalFinishers = len(teamResults) # base team points @@ -123,21 +125,33 @@ def calculate_competition_points(results, teams, event): competitionAgeCats = get_competition_agecats(event) competitionPoints = {} + # reference = { 'penalty':{}, 'participants':{}, 'counters':{}} + reference = {} + reference['total_participants'] = len(results) # for each gender competition for gender in GENDER_COMPETITIONS: competitionPoints[gender] = {} + + # reference['penalty'][gender] = {} + # reference['participants'][gender] = {} # for each age category in each set of results for ageCat in competitionAgeCats: logging.info(f"Processing {gender}:{ageCat} in {event}") + if ageCat not in reference: + reference[ageCat] = {} + + reference[ageCat][gender] = {} + maxCounters = get_team_counters(ageCat=ageCat) competitionPoints[gender][ageCat] = None if ageCat == OVERALL: # For overall competition, don't filter on ageCat - ageCatResults = extract_filtered_results(results=results, gender=gender) + ageCatResults = extract_filtered_results( + results=results, gender=gender) else: # get the results for that age category ageCatResults = extract_filtered_results( @@ -150,6 +164,11 @@ def calculate_competition_points(results, teams, event): # what the penalty points are penaltyPoints = totalParticipants + PENALTY_POINTS + # build reference structure + reference[ageCat][gender]['penalty'] = penaltyPoints + reference[ageCat][gender]['participants'] = totalParticipants + reference[ageCat][gender]['counters'] = maxCounters + # Work through all the teams we know about for team in teams.index: # get their results @@ -170,7 +189,8 @@ def calculate_competition_points(results, teams, event): competitionPoints[gender][ageCat] = teamResult else: competitionPoints[gender][ageCat] = pd.concat( - [competitionPoints[gender][ageCat], teamResult], ignore_index=0 + [competitionPoints[gender][ageCat], + teamResult], ignore_index=0 ) points = tidy_points( @@ -188,7 +208,7 @@ def calculate_competition_points(results, teams, event): # remove the working data del competitionPoints[gender][ageCat] - return competitionPoints + return (reference, competitionPoints) def get_all_possible_columns(results): diff --git a/tools/src/adapter_points.py.bkup b/tools/src/adapter_points.py.bkup deleted file mode 100644 index d206587..0000000 --- a/tools/src/adapter_points.py.bkup +++ /dev/null @@ -1,249 +0,0 @@ -import pandas as pd -import numpy as np -import logging -from .utils_consts import ( - SeniorAgeCats, - PENALTY_POINTS, - AgeCatCounterOverrides, - DEFAULT_COUNTERS, - CompetitionsBreakouts, - GENDER_COMPETITIONS, - NONBINARY, - OVERALL -) - - -def get_team_counters(ageCat): - if ageCat in AgeCatCounterOverrides: - return AgeCatCounterOverrides[ageCat] - else: - return DEFAULT_COUNTERS - - -def get_competition_breakout(event): - if event in CompetitionsBreakouts: - return CompetitionsBreakouts[event] - else: - return CompetitionsBreakouts["default"] - - -def get_competition_agecats(event): - # from eg. U20-Senior-Masters_Combined - # return a list of agecats in upper case - if event is not None: - agepart = event.upper().split("_")[0] - if "-" in agepart: - agecats = agepart.split("-") - else: - agecats = [agepart] - - # normalise filename MASTERS to MASTER to match the agecat - if "MASTERS" in agecats: - agecats.remove("MASTERS") - agecats.append("MASTER") - - return agecats - else: - raise Exception("get_competition_agecats: event is None") - - -def extract_team_results(results, team): - # we want a DataFrame back - return results[results["clubnumber"] == team] - - -def extract_filtered_results(results=None, ageCat=None, gender=None): - - if results is None: - raise Exception("Cannot filter empty result set") - - # get Age Category results - if ageCat is not None: - ageCatResults = results[results["AgeCat"] == ageCat] - else: - ageCatResults = results - - # extract the matching gender records - if gender is not None: - genderResults = ageCatResults[ageCatResults["gender"] == gender].reset_index() - - # preserve the finish position column - genderResults["finishPosition"] = genderResults["position"] - - genderResults["position"] = genderResults.index + 1 - else: - genderResults = ageCatResults - - return genderResults - - -def calculate_team_points(team, teamResults, maxCounters, penaltyPoints): - # sum up the points for first N counters - countingResults = teamResults.head(maxCounters) - - # how many results? - teamCounters = len(countingResults) - - # what were the finisher positions - teamFinishers = ",".join( - teamResults.head(maxCounters)["position"].astype("string").values.tolist() - ) - totalFinishers = len(teamResults) - # base team points - teamPoints = countingResults["position"].sum() - - teamPenalty = 0 - if teamCounters < maxCounters: - # fewer than the maxCounters, so calculate penalty - teamPenalty = (maxCounters - teamCounters) * penaltyPoints - - teamResult = { - "team": team, - "finisherPositions": teamFinishers, - "teamPoints": teamPoints, - "penaltyPoints": teamPenalty, - "totalPoints": teamPoints + teamPenalty, - "totalFinishers": totalFinishers, - } - # return a tuple of total points, counters used, and penalty part - # return (countingResults.sum()+teamPenalty, teamCounters, teamPenalty) - df = pd.DataFrame(teamResult, index=[team]) - return df - - -def calculate_competition_points(results, teams, event): - totalRunners = len(event) - - # What competitions are in these results - competitionBreakout = get_competition_breakout(event) - - competitionAgeCats = get_competition_agecats(event) - - competitionPoints = {} - - # for each gender competition - for gender in GENDER_COMPETITIONS: - competitionPoints[gender] = {} - competitionPoints[gender][OVERALL] = None - - # for each age category in each set of results - for ageCat in competitionAgeCats: - - maxCounters = get_team_counters(ageCat=ageCat) - - competitionPoints[gender][ageCat] = None - - # get the results for that age category - ageCatResults = extract_filtered_results( - results=results, ageCat=ageCat, gender=gender - ) - - # get the results for the gender specific competition - genderResults = extract_filtered_results( results=results, gender=gender) - - # how many took part - totalParticipants = len(ageCatResults) - totalGenderParticipants = len(genderResults) - - # what the penalty points are - penaltyPoints = totalParticipants + PENALTY_POINTS - penaltyGenderPoints = totalGenderParticipants + PENALTY_POINTS - - # Work through all the teams we know about - for team in teams.index: - # get their results - teamResults = extract_team_results(ageCatResults, team) - genderTeamResults = extract_team_results(genderResults, team) - - # work out their points, and how many constituted - teamResult = calculate_team_points( - team=team, - teamResults=teamResults, - maxCounters=maxCounters, - penaltyPoints=penaltyPoints, - ) - genderTeamResult = calculate_team_points( - team=team, - teamResults=genderTeamResults, - maxCounters=maxCounters, - penaltyPoints=penaltyGenderPoints, - ) - - # note the gender of the final results so it appears in final - teamResult["gender"] = gender - - if competitionPoints[gender][ageCat] is None: - competitionPoints[gender][ageCat] = teamResult - else: - competitionPoints[gender][ageCat] = pd.concat( - [competitionPoints[gender][ageCat], teamResult], ignore_index=0 - ) - - if competitionPoints[gender][OVERALL] is None: - competitionPoints[gender][OVERALL] = genderTeamResults - else: - competitionPoints[gender][OVERALL] = pd.concat( - [competitionPoints[gender][OVERALL], genderTeamResult], ignore_index=0 - ) - - points = tidy_points( - competitionPoints[gender][ageCat] - .sort_values(by="totalPoints") - .join(other=teams, on="team") - ) - - overallPoints = tidy_points( - competitionPoints[gender][OVERALL].sort_values(by="totalPoints") - .join(other=teams, on="team") - ) - - # is there a points table - if points is not None: - competitionPoints[gender][ageCat] = points - else: - # remove the working data - del competitionPoints[gender][ageCat] - - if overallPoints is not None: - competitionPoints[gender][OVERALL] = overallPoints - else: - del competitionPoints[gender][OVERALL] - - - - return competitionPoints - - -def get_all_possible_columns(results): - idealCols = [ - "Club name", - "team", - "gender", - "finisherPositions", - "teamPoints", - "penaltyPoints", - "totalPoints", - "totalFinishers", - "Website", - ] - columns = [] - for col in idealCols: - if col in results: - columns.append(col) - return columns - - -def tidy_points(results=None): - if results is not None: - # check if there are no results in the data set - if results["totalFinishers"].sum() == 0: - return None - availCols = get_all_possible_columns(results) - provisionalResults = results[availCols] - provisionalResults.insert( - loc=0, column="position", value=np.arange(len(results)) + 1 - ) - returnResults = provisionalResults[provisionalResults["totalFinishers"] > 0] - return returnResults - else: - return None diff --git a/tools/src/adapter_team_results.py b/tools/src/adapter_team_results.py new file mode 100644 index 0000000..4fee169 --- /dev/null +++ b/tools/src/adapter_team_results.py @@ -0,0 +1,55 @@ +import pandas as pd +import numpy as np +from .utils_consts import CLUB_PARSED, DATA_DIR +import glob +import os +from .utils_functions import fetch_results_filenames +import logging + + +def load_team_results(dir: str = None): + teamResults = fetch_results_filenames(dir) + eventResults = {} + for race in teamResults: + logging.info(f"Loading {race['filename']}") + teamResult = pd.read_csv(race['filename']) + if race['agecat'] not in eventResults: + eventResults[race['agecat']] = {} + eventResults[race['agecat']][race['gender']] = teamResult + + return eventResults + + +def extract_race_results(allEvents: dict = None, requiredCompetition: str = None, requiredGender: str = None): + """ Fetch ALL of the relevant races from the structure, return as list """ + results = {} + for event in allEvents: + # for competition in allEvents[event]: + # for gender in allEvents[event][competition]: + # if gender == requiredGender and competition == requiredCompetition: + results[event] = allEvents[event][requiredCompetition][requiredGender] + return results + + +def calculate_team_standings(raceResults: dict = None, eventMeta: dict = None,competition:str = None, gender:str = None): + """ For the results we have, roll up the results """ + table = {} + for race in raceResults: + for index, teamResult in raceResults[race].iterrows(): + team = teamResult['team'] + teamPoints = int(teamResult['totalPoints']) + clubName = teamResult['Club name'] + if race not in table: + table[race] = {} + + table[race][team] = teamPoints + + results = pd.DataFrame(table) + + results['Total'] = 0 + for race in raceResults: + raceMeta = eventMeta[race]['races'][competition][gender] + results.iloc[:, race-1] = results.iloc[:, race-1].replace(np.nan, raceMeta['penalty']).astype(int) + results['Total'] = results['Total']+results.iloc[:, race-1].astype(int) + + return results.sort_values(by=['Total']) diff --git a/tools/src/utils_consts.py b/tools/src/utils_consts.py index 9effdcc..535b27f 100644 --- a/tools/src/utils_consts.py +++ b/tools/src/utils_consts.py @@ -83,10 +83,13 @@ # hard-coded for now YEAR="2022-23" EVENT="2" -DATA_DIR = "data/2022-23/1" +DATA_DIR = "data/2022-23/2" CLUB_SUBMISSIONS = "club-submissions" CLUB_PARSED = "club-parsed" -RESULTS_DIR = "results/provisional/2022-23/1" +RESULTS_DIR = "results/provisional/2022-23/2" TEAMS = "data/reference/clubs.csv" GENDERS = "data/reference/genders.csv" -ADJUSTMENTS_DIR = "data/adjustments/2022-23/1" +ADJUSTMENTS_DIR = "data/adjustments/2022-23/2" + + +BASE_RESULTS = "results/provisional" \ No newline at end of file diff --git a/tools/src/utils_functions.py b/tools/src/utils_functions.py index 5285139..9df9e3d 100644 --- a/tools/src/utils_functions.py +++ b/tools/src/utils_functions.py @@ -21,3 +21,21 @@ def fetch_volunteers_from_dir(dir: str = None): return volunteers[0] else: return None + +def fetch_events( dir: str = None): + if dir is not None: + events = glob.glob(dir + "?") + return events + else: + return None + +def fetch_results_filenames(dir:str = None): + results = None + if dir is not None: + results = [] + for result in glob.glob(dir + "/*.team.results.csv"): + filename = os.path.basename(result) + (eventname,agecat,gender) = filename.split(".")[:3] + + results.append({'filename':dir+"/"+filename,'eventname':eventname,'agecat':agecat,'gender':gender}) + return results \ No newline at end of file diff --git a/tools/team-results.py b/tools/team-results.py new file mode 100755 index 0000000..860cdd7 --- /dev/null +++ b/tools/team-results.py @@ -0,0 +1,87 @@ +#!env python + +from dotenv import dotenv_values +import logging +from src.adapter_sheets import load_volunteers, load_results +from src.adapter_json import json_load +from src.adapter_format import export_results +from src.utils_consts import ( + DATA_DIR, + TEAMS, + EXT_TIMES, + EXT_PLACES, + EXT_META, + EXT_ADJUSTMENTS, + EXT_CSV, + GENDERS, + RESULTS_DIR, + GENDER_COMPETITION_MAP, + HTML_DIR, + MARKDOWN_DIR, + YEAR, + EVENT, + BASE_RESULTS +) +from src.utils_functions import fetch_events +from src.adapter_gender import gender_process +from src.adapter_team import process_teams, load_team_submissions +from src.adapter_results import results_merge, tidy_results, merge_runners +from src.adapter_times import adjust_times +from src.adapter_places import adjust_places, process_final_results +from src.adapter_points import calculate_competition_points +from src.adapter_pretty_html import render +from src.adapter_team_results import load_team_results, extract_race_results, calculate_team_standings +import pandas as pd +import pathlib +from pretty_html_table import build_table + +logging.basicConfig(level=logging.DEBUG) +config = dotenv_values(".env") + +# hard-coded to same location for now +teams = pd.read_csv(TEAMS, index_col="Number") +genders = pd.read_csv(GENDERS, index_col="shortcode") + +eventDirectories = fetch_events( + dir=BASE_RESULTS+"/"+config['PROCESS_YEAR']+"/") + +events = {} +event_meta = {} +# Load the team results from available events +for event in eventDirectories: + eventNumber = int(event.split("/")[-1:][0]) + logging.info(f"Processing: {event}") + events[eventNumber] = load_team_results(dir=event) + event_meta[eventNumber] = json_load(filename=event+"/meta.json") + +baseEvent = list(events.keys())[0] + +theFiles = [] +for competition in events[baseEvent]: + for gender in GENDER_COMPETITION_MAP: + logging.info( + f"Processing {competition}:{GENDER_COMPETITION_MAP[gender]}") + raceResults = extract_race_results( + allEvents=events, requiredCompetition=competition, requiredGender=GENDER_COMPETITION_MAP[gender]) + + teamStandings = calculate_team_standings( + raceResults=raceResults, eventMeta=event_meta, competition=competition, gender=gender) + + normalisedTeamStandings = teamStandings.join(other=teams) + resultPages = export_results(results=normalisedTeamStandings, + base_file_name=f"{competition}_{gender}", suffix=".team.standings") + + theFiles.append(resultPages["html"].split("/")[-1:][0]) + + +# Render a basic HTML index + +def make_clickable(val): + return f'{val}' + + +filesDF = pd.DataFrame({'teamStandings': theFiles}) +filesDF = filesDF.style.format({'teamStandings': make_clickable}) + +filesDF.to_html(RESULTS_DIR+HTML_DIR+"/"+"teamStandings.html", + index=False, render_links=True, escape=False)