Updates/nces school updates

dbt/models/marts/schools/_schools__models.yml

@@ -9,6 +9,59 @@ models:
           combination_of_columns:
             - school_year
             - school_id
+    columns: 
+      - name: school_id
+        description: the NCES id associated with a school 
+        data_tests:
+          - not_null
+      - name: school_year
+        description: school year for which the stats are relevant (most schools will have multiple school years in this table, and the data may change every year)
+      - name: survey_year
+        description: what NCES survey year included these statistics
+      - name: first_survey_year
+        description: the first year the school ID was part of our NCES school upload. NULL values indicate a school was manually entered and has never matched an NCES entry
+      - name: is_stage_el
+        description: binary; 1 if the school has any grade levels 0-5, 0 if not 
+      - name: is_stage_mi
+        description: binary; 1 if the school has any grade levels 6-8, 0 if not 
+      - name: is_stage_hi
+        description: binary; 1 if the school has any grade levels 9-12, 0 if not 
+      - name: is_rural
+        description: school community is considered rural_distant, rural_town, rural_fringe, town_remote, or town_distant
+      - name: title_i_status
+        description: the detailed mapping of title I status options.
+          1 = Title I targeted assistance eligible school-No program
+          2 = Title I targeted assistance school
+          3 = Title I schoolwide eligible-Title I targeted assistance program
+          4 = Title I schoolwide eligible school-No program
+          5 = Title I schoolwide school
+          6 = not a Title I school 
+      - name: is_title_i
+        description: binary, whether a school is eligible for title I or not (1-5 in the detailed title_i_status field). NULL usually indicates a private school.
+      - name: community type
+        description: NCES community category
+      - name: school_category
+        description: Regular School, Alternative School, Special Education School, Career and Technical School, or NULL
+      - name: school_type
+        description: public, private, charter, or NULL
+      - name: is_high_needs
+        description: binary, whether the percentage of FRL (free and reduced lunch) students at the school is >50%
+      - name: open_status
+        description: 1-open, 2-closed, 3-new, 4-added, 5-changed boundary/agency, 6-inactive, 7-future, 8-reopened, or NULL
+      - name: is_school_open
+        description: binary, captures whether school is open (options 1,3,4,5, and 8 in open_status) or closed (options 2,6,and 7 in open_status)
+      - name: created_at
+        description: when the record was created for that school in our systems (not when the school was actually opened)
+      - name: updated_at
+        description: when the record was most recently updated in our systems
+      - name: urg_percent
+        description: the share of students with race reported who are from an underrepresented racial group; excludes "two or more races" from the calculation entirely
+      - name: urg_no_tr_numerator_percent
+        description: the share of students with race reported who are from an underrepresented racial group; excludes "two or more races" from the numerator only; used for AFE reporting 
+      - name: urg_with_tr_percent
+        description: the share of students with race reported who are from an underrepresented racial group; includes "two or more races" in both the numerator and denominator
+      - name: total_urg_no_tr_students
+        description: total number of URG students, excluding those identifying as two or more races
     config:
       tags: ['released']
 
@@ -50,19 +103,19 @@ models:
         data_tests:
           - not_null
           - unique
-
-      - name: urg_percent
-        description: the share of students with race reported who are from an underrepresented racial group; excludes "two or more races" from the calculation entirely
-
-      - name: urg_no_tr_numerator_percent
-        description: the share of students with race reported who are from an underrepresented racial group; excludes "two or more races" from the numerator only; used for AFE reporting 
-
-      - name: urg_with_tr_percent
-        description: the share of students with race reported who are from an underrepresented racial group; includes "two or more races" in both the numerator and denominator
-
-      - name: last_survey_year 
-        description: the school year associated with the most recent information about this school from NCES
 
+      - name: city
+        description: city where the school is located
+
+      - name: state
+        description: state where the school is located
+
+      - name: zip
+        description: zip code of the school address
+
+      - name: last_survey_year
+        description: the most recent year the school ID was part of the NCES school upload. NULL values indicate a school was manually entered and has never matched an NCES entry
+
       - name: is_stage_el
         description: binary; 1 if the school has any grade levels 0-5, 0 if not 
 
@@ -71,6 +124,54 @@ models:
 
       - name: is_stage_hi
         description: binary; 1 if the school has any grade levels 9-12, 0 if not 
+
+      - name: is_rural
+        description: school community is considered rural_distant, rural_town, rural_fringe, town_remote, or town_distant
+
+      - name: title_i_status
+        description: the detailed mapping of title I status options.
+          1 = Title I targeted assistance eligible school-No program
+          2 = Title I targeted assistance school
+          3 = Title I schoolwide eligible-Title I targeted assistance program
+          4 = Title I schoolwide eligible school-No program
+          5 = Title I schoolwide school
+          6 = not a Title I school 
+
+      - name: is_title_i
+        description: binary, whether a school is eligible for title I or not (1-5 in the detailed title_i_status field). NULL usually indicates a private school.
+
+      - name: community type
+        description: NCES community category
+
+      - name: school_category
+        description: Regular School, Alternative School, Special Education School, Career and Technical School, or NULL
+
+      - name: school_type
+        description: public, private, charter, or NULL
+
+      - name: is_high_needs
+        description: binary, whether the percentage of FRL (free and reduced lunch) students at the school is >50%
+
+      - name: open_status
+        description: 1-open, 2-closed, 3-new, 4-added, 5-changed boundary/agency, 6-inactive, 7-future, 8-reopened, or NULL
+
+      - name: is_school_open
+        description: binary, captures whether school is open (options 1,3,4,5, and 8 in open_status) or closed (options 2,6,and 7 in open_status)
+
+      - name: school_created_at
+        description: when the record was created for that school in our systems (not when the school was actually opened)
+
+      - name: school_last_updated_at
+        description: when the record was most recently updated in our systems
+
+      - name: urg_percent
+        description: the share of students with race reported who are from an underrepresented racial group; excludes "two or more races" from the calculation entirely
+
+      - name: urg_no_tr_numerator_percent
+        description: the share of students with race reported who are from an underrepresented racial group; excludes "two or more races" from the numerator only; used for AFE reporting 
+
+      - name: urg_with_tr_percent
+        description: the share of students with race reported who are from an underrepresented racial group; includes "two or more races" in both the numerator and denominator
 
       - name: school_level_simple
         description: a combination of all school levels the school contains, separated by underscores

dbt/models/marts/schools/dim_school_stats_by_years.sql

@@ -1,4 +1,10 @@
--- Note: this data is built using NCES data recevied
+/*
+Note: this data is built using NCES data furnished by RED and uploaded once a year by the acquisitions team
+
+Edits:
+- CK - May 2025 - Added counts by gender, title i status, and open status
+*/
+
 
 with 
 school_stats_by_years as (
@@ -38,7 +44,7 @@ school_stats_by_years as (
 
     from {{ ref('stg_dashboard__school_stats_by_years') }}
 
-    {{ dbt_utils.group_by(40) }}
+    {{ dbt_utils.group_by(44) }}
 ),
 
 combined as (

dbt/models/marts/schools/dim_schools.sql

@@ -1,29 +1,37 @@
--- Model: dim_schools
--- Scope: all dimensions we have/need for schools; one row per school + school_year
+/*
+Model: dim_schools
+Scope: all dimensions we have/need for schools; one row per school, most recent data. 
+
+Note on schools with NULL last_survey_year - there are 5500 schools wth no last_survey_year. The vast majority were from manual uploads in 2017-2018. These schools have no valid NCES match. However, teachers are still selecting them from the dropdown list. Over 150 of those schools were active in 2024-25. Until those schools are removed from the dropdown list and teachers are prompted to reaffiliate, we will keep them in the list of schools for now. 
+
+Edit log
+- CK - May 2025 - Added county ID and name, count by gender, open status, and title I detail
+*/
+
+
 with
 schools as (
     select *
     from {{ ref('stg_dashboard__schools') }}
-),
+)
 
-school_districts as (
+, school_districts as (
     select *
     from {{ ref('stg_dashboard__school_districts') }}
-),
+)
 
-school_stats_by_years as (
+, school_stats_by_years as (
     select *, 
         row_number() 
             over(
                 partition by school_id 
                 order by school_year desc) as row_num
 
     from {{ ref('dim_school_stats_by_years') }}
-),
+)
 
-combined as (
+, combined as (
     select
-        -- schools
         schools.school_id,
         schools.city,
         schools.state,
@@ -41,6 +49,7 @@ combined as (
         ) as school_level_simple,
 
         school_stats_by_years.is_rural,
+        school_stats_by_years.title_i_status,
         school_stats_by_years.is_title_i,
         school_stats_by_years.community_type,
 
@@ -57,8 +66,14 @@ combined as (
         school_districts.school_district_id,
         school_districts.school_district_name,
 
+        --county
+        schools.county_id,
+        schools.county_name,
+
         -- nces school metrics
         school_stats_by_years.total_students,
+        school_stats_by_years.count_student_female,
+        school_stats_by_years.count_student_male,
         school_stats_by_years.count_student_am,
         school_stats_by_years.count_student_as,
         school_stats_by_years.count_student_hi,
@@ -75,11 +90,17 @@ combined as (
         school_stats_by_years.urg_with_tr_percent,
         school_stats_by_years.urg_no_tr_numerator_percent,
         school_stats_by_years.frl_eligible_percent,
+
+
+        --open status
+        school_stats_by_years.open_status,
+        school_stats_by_years.is_school_open,
 
         -- dates 
         min(schools.created_at) as school_created_at,
         max(schools.updated_at) as school_last_updated_at
 
+
     from schools
     left join school_stats_by_years
         on schools.school_id = school_stats_by_years.school_id
@@ -88,7 +109,7 @@ combined as (
     left join school_districts
         on schools.school_district_id = school_districts.school_district_id
 
-    {{ dbt_utils.group_by(36) }}
+    {{ dbt_utils.group_by(43) }}
 )
 
 select *

dbt/models/staging/dashboard/base/base_dashboard__school_stats_by_years.sql

@@ -2,50 +2,7 @@ with
 source as (
     select * 
     from {{ source('dashboard', 'school_stats_by_years') }}
-),
-
-renamed as (
-    select
-        school_id,
-        school_year,
-
-        lower(grades_offered_lo) as grades_offered_lo,
-        lower(grades_offered_hi) as grades_offered_hi,
-
-        grade_pk_offered    as is_grade_pk,
-        grade_kg_offered    as is_grade_kg,
-        grade_01_offered    as is_grade_01,
-        grade_02_offered    as is_grade_02,
-        grade_03_offered    as is_grade_03,
-        grade_04_offered    as is_grade_04,
-        grade_05_offered    as is_grade_05,
-        grade_06_offered    as is_grade_06,
-        grade_07_offered    as is_grade_07,
-        grade_08_offered    as is_grade_08,
-        grade_09_offered    as is_grade_09,
-        grade_10_offered    as is_grade_10,
-        grade_11_offered    as is_grade_11,
-        grade_12_offered    as is_grade_12,
-        grade_13_offered    as is_grade_13,
-
-        community_type,
-        virtual_status,
-        title_i_status,
-
-        coalesce(students_total,0)      as total_students,
-        coalesce(student_am_count,0)    as count_student_am,
-        coalesce(student_as_count,0)    as count_student_as,
-        coalesce(student_hi_count,0)    as count_student_hi,
-        coalesce(student_bl_count,0)    as count_student_bl,
-        coalesce(student_wh_count,0)    as count_student_wh,
-        coalesce(student_hp_count,0)    as count_student_hp,
-        coalesce(student_tr_count,0)    as count_student_tr,
-        coalesce(frl_eligible_total,0)  as total_frl_eligible_students,
-
-        created_at,
-        updated_at
-    from source
 )
 
 select * 
-from renamed
+from source

dbt/models/staging/dashboard/base/base_dashboard__schools.sql

@@ -2,28 +2,7 @@ with
 source as (
     select * 
     from {{ source('dashboard', 'schools') }}
-),
-
-renamed as (
-    select
-        id                  as school_id,
-        school_district_id,
-        name                as school_name,
-        city,
-        state,
-        zip,
-        school_type,
-        created_at,
-        updated_at,
-        address_line1,
-        address_line2,
-        address_line3,
-        latitude,
-        longitude,
-        school_category,
-        last_known_school_year_open
-    from source
 )
 
 select * 
-from renamed
+from source