diff --git a/README.rdoc b/README.rdoc index 56feadadec..ef736240e6 100644 --- a/README.rdoc +++ b/README.rdoc @@ -1,4 +1,7 @@ = Species+ + +NOTE: There is a separate {public API application}[https://github.com/unepwcmc/species-api] for Species+ + {}[http://travis-ci.org/unepwcmc/SAPI] {}[https://codeclimate.com/github/unepwcmc/SAPI] {Coverage Status}[https://coveralls.io/r/unepwcmc/SAPI?branch=master] diff --git a/app/assets/javascripts/species/controllers/downloads_controller.js.coffee b/app/assets/javascripts/species/controllers/downloads_controller.js.coffee index 7682ecd1ee..f488c2dfe8 100644 --- a/app/assets/javascripts/species/controllers/downloads_controller.js.coffee +++ b/app/assets/javascripts/species/controllers/downloads_controller.js.coffee @@ -1,7 +1,7 @@ Species.DownloadsController = Ember.Controller.extend Species.Spinner, needs: [ 'downloadsForCmsListings', - 'downloadsForCitesListings', 'downloadsForCitesRestrictions', + 'downloadsForCitesListings', 'downloadsForCitesRestrictions', 'downloadsForCitesProcesses', 'downloadsForEuListings', 'downloadsForEuDecisions' ] downloadsPopupVisible: false @@ -23,6 +23,9 @@ Species.DownloadsController = Ember.Controller.extend Species.Spinner, legislationIsCitesRestrictions: ( -> @get('citesLegislation') == 'restrictions' ).property('citesLegislation') + legislationIsCitesProcesses: ( -> + @get('citesLegislation') == 'processes' + ).property('citesLegislation') legislationIsEuListings: ( -> @get('euLegislation') == 'listings' ).property('euLegislation') diff --git a/app/assets/javascripts/species/controllers/downloads_for_cites_listings_controller.js.coffee b/app/assets/javascripts/species/controllers/downloads_for_cites_listings_controller.js.coffee index 120ea591d8..5f8f9a993e 100644 --- a/app/assets/javascripts/species/controllers/downloads_for_cites_listings_controller.js.coffee +++ b/app/assets/javascripts/species/controllers/downloads_for_cites_listings_controller.js.coffee @@ -25,7 +25,7 @@ Species.DownloadsForCitesListingsController = Ember.Controller.extend } ).filter((e) -> e.taxonConcepts.length > 0 - ) + ) else @get('higherTaxaController.contentByRank') ).property('higherTaxaController.contentByRank.@each', 'taxonConceptQuery') @@ -103,4 +103,4 @@ Species.DownloadsForCitesListingsController = Ember.Controller.extend @set('selectedTaxonConcepts', []) deleteGeoEntitySelection: (context) -> - @get('selectedGeoEntities').removeObject(context) \ No newline at end of file + @get('selectedGeoEntities').removeObject(context) diff --git a/app/assets/javascripts/species/controllers/downloads_for_cites_processes_controller.js.coffee b/app/assets/javascripts/species/controllers/downloads_for_cites_processes_controller.js.coffee new file mode 100644 index 0000000000..8012dcbd8a --- /dev/null +++ b/app/assets/javascripts/species/controllers/downloads_for_cites_processes_controller.js.coffee @@ -0,0 +1,123 @@ +Species.DownloadsForCitesProcessesController = Ember.Controller.extend + designation: 'cites' + + needs: ['geoEntities','higherTaxaCitesEu', 'downloads'] + + higherTaxaController: ( -> + @get('controllers.higherTaxaCitesEu') + ).property() + + geoEntityQuery: null + taxonConceptQuery: null + selectedGeoEntities: [] + selectedTaxonConcepts: [] + timeScope: 'current' + timeScopeIsCurrent: ( -> + @get('timeScope') == 'current' + ).property('timeScope') + years: [1975..new Date().getFullYear()] + selectedYears: [] + processType: 'Both' + documentTypeIsCitesSuspensions: ( -> + @get('documentType') == 'CitesSuspensions' + ).property('documentType') + + autoCompleteTaxonConcepts: ( -> + if @get('taxonConceptQuery') && @get('taxonConceptQuery').length > 0 + re = new RegExp("^"+@get('taxonConceptQuery'),"i") + @get('higherTaxaController.contentByRank') + .map((e) => + { + rankName: e.rankName + taxonConcepts: e.taxonConcepts.filter((item) => + re.test item.get('fullName') + ) + } + ).filter((e) -> + e.taxonConcepts.length > 0 + ) + else + @get('higherTaxaController.contentByRank') + ).property('higherTaxaController.contentByRank.@each', 'taxonConceptQuery') + + autoCompleteRegions: ( -> + if @get('geoEntityQuery') && @get('geoEntityQuery').length > 0 + re = new RegExp("(^|\\(| )"+@get('geoEntityQuery'),"i") + @get('controllers.geoEntities.regions') + .filter (item, index, enumerable) => + re.test item.get('name') + else + @get('controllers.geoEntities.regions') + ).property('controllers.geoEntities.regions.@each', 'geoEntityQuery') + + autoCompleteCountries: ( -> + if @get('geoEntityQuery') && @get('geoEntityQuery').length > 0 + re = new RegExp("(^|\\(| )"+@get('geoEntityQuery'),"i") + @get('controllers.geoEntities.countries') + .filter (item, index, enumerable) => + re.test item.get('name') + else + @get('controllers.geoEntities.countries') + ).property('controllers.geoEntities.countries.@each', 'geoEntityQuery') + + selectedGeoEntitiesIds: ( -> + @get('selectedGeoEntities').mapProperty('id') + ).property('selectedGeoEntities.@each') + + selectedTaxonConceptsIds: ( -> + @get('selectedTaxonConcepts').mapProperty('id') + ).property('selectedTaxonConcepts.@each') + + toParams: ( -> + { + data_type: 'Processes' + filters: + process_type: @get('processType') + designation: @get('designation') + geo_entities_ids: @get('selectedGeoEntitiesIds') + taxon_concepts_ids: @get('selectedTaxonConceptsIds') + set: @get('timeScope') + years: @get('selectedYears') + csv_separator: @get('controllers.downloads.csvSeparator') + } + ).property( + 'selectedGeoEntitiesIds.@each', 'selectedTaxonConceptsIds.@each', + 'timeScope', 'selectedYears.@each', 'processType', 'controllers.downloads.csvSeparator' + ) + + downloadUrl: ( -> + '/species/exports/download?' + $.param(@get('toParams')) + ).property('toParams') + + actions: + startDownload: () -> + @set('downloadInProgress', true) + @set('downloadMessage', 'Downloading...') + $.ajax({ + type: 'GET' + dataType: 'json' + url: @get('downloadUrl') + }).done((data) => + @set('downloadInProgress', false) + if data.total > 0 + @set('downloadMessage', null) + ga('send', { + hitType: 'event', + eventCategory: 'Downloads: ' + @get('processType'), + eventAction: 'Format: CSV', + eventLabel: @get('controllers.downloads.csvSeparator') + }) + window.location = @get('downloadUrl') + return + else + @set('downloadMessage', 'No results') + ) + + deleteTaxonConceptSelection: (context) -> + @set('selectedTaxonConcepts', []) + + deleteGeoEntitySelection: (context) -> + @get('selectedGeoEntities').removeObject(context) + + deleteYearSelection: (context) -> + @get('selectedYears').removeObject(Number(context)) diff --git a/app/assets/javascripts/species/controllers/elibrary_search_controller.js.coffee b/app/assets/javascripts/species/controllers/elibrary_search_controller.js.coffee index 0ebb2e3508..846b0ab3dd 100644 --- a/app/assets/javascripts/species/controllers/elibrary_search_controller.js.coffee +++ b/app/assets/javascripts/species/controllers/elibrary_search_controller.js.coffee @@ -33,7 +33,8 @@ Species.ElibrarySearchController = Ember.Controller.extend Species.Spinner, allDocumentTypes = @get('controllers.events.documentTypes') .concat @get('controllers.events.interSessionalDocumentTypes') .concat @get('controllers.events.identificationDocumentTypes') - + if @get('isSignedIn') + allDocumentTypes = allDocumentTypes.concat(@get('controllers.events.interSessionalNonPublicDocumentTypes')) @set('selectedDocumentType', allDocumentTypes.findBy('id', filtersHash.document_type)) general_subtype_type = @get_general_subtype_type(filtersHash) @@ -71,10 +72,10 @@ Species.ElibrarySearchController = Ember.Controller.extend Species.Spinner, general_subtype: isGeneralSubType } - getDocTypeParam: -> + getDocTypeParam: -> id = @get('selectedDocumentType.id') - if id != '__all__' then id else null + if id != '__all__' then id else null filteredDocumentTypes: ( -> if @get('selectedEventType') diff --git a/app/assets/javascripts/species/templates/downloads_for_cites.handlebars b/app/assets/javascripts/species/templates/downloads_for_cites.handlebars index 14df63ce5d..b30c018532 100644 --- a/app/assets/javascripts/species/templates/downloads_for_cites.handlebars +++ b/app/assets/javascripts/species/templates/downloads_for_cites.handlebars @@ -4,11 +4,16 @@ LISTINGS {{/view}} -
  • +
  • {{#view Species.ToggleButton option="restrictions" valueBinding="controller.citesLegislation"}} QUOTAS/SUSPENSIONS {{/view}}
  • +
  • + {{#view Species.ToggleButton option="processes" valueBinding="controller.citesLegislation"}} + Processes + {{/view}} +
  • {{#view Species.DownloadsForLegislation isVisibleBinding="controller.legislationIsCitesListings" @@ -136,3 +141,84 @@ {{/view}} +{{#view Species.DownloadsForLegislation + isVisibleBinding="controller.legislationIsCitesProcesses" + controllerBinding="controllers.downloadsForCitesProcesses" +}} +
    +
    +
    + +
    +
    + +
    +
    + +
    +
    +
    +
    +
    + +
    +
    + {{#if timeScopeIsCurrent}} + Only current data has
    been selected. + {{else}} + {{#unless documentTypeIsCitesSuspensions}} + + {{/unless}} + {{/if}} +
    +
    +
    +
    + +
    +
    + +
    +
    +
    + {{view Species.StartDownloadButton controllerBinding="controller"}} +
    + {{#if controller.downloadInProgress}} + + {{/if}} + {{controller.downloadMessage}} +
    +
    +{{/view}} diff --git a/app/assets/javascripts/species/templates/taxon_concept.handlebars b/app/assets/javascripts/species/templates/taxon_concept.handlebars index e403f38597..47d8e14822 100644 --- a/app/assets/javascripts/species/templates/taxon_concept.handlebars +++ b/app/assets/javascripts/species/templates/taxon_concept.handlebars @@ -8,7 +8,7 @@ {{/if}}

    {{fullName}}

    -

    {{authorYear}}

    +

    {{authorYear}}

    diff --git a/db/migrate/20230803181442_remove_language_restrictions_from_autocomplete_view.rb b/db/migrate/20230803181442_remove_language_restrictions_from_autocomplete_view.rb new file mode 100644 index 0000000000..192839d594 --- /dev/null +++ b/db/migrate/20230803181442_remove_language_restrictions_from_autocomplete_view.rb @@ -0,0 +1,15 @@ +class RemoveLanguageRestrictionsFromAutocompleteView < ActiveRecord::Migration + def up + execute "DROP VIEW IF EXISTS auto_complete_taxon_concepts_view" + execute "CREATE VIEW auto_complete_taxon_concepts_view AS #{view_sql('20230803181442', 'auto_complete_taxon_concepts_view')}" + execute File.read(File.expand_path('../../mviews/011_rebuild_auto_complete_taxon_concepts_mview.sql', __FILE__)) + execute "SELECT * FROM rebuild_auto_complete_taxon_concepts_mview()" + end + + def down + execute "DROP VIEW IF EXISTS auto_complete_taxon_concepts_view" + execute "CREATE VIEW auto_complete_taxon_concepts_view AS #{view_sql('20220808123526', 'auto_complete_taxon_concepts_view')}" + execute File.read(File.expand_path('../../mviews/011_rebuild_auto_complete_taxon_concepts_mview.sql', __FILE__)) + execute "SELECT * FROM rebuild_auto_complete_taxon_concepts_mview()" + end +end diff --git a/db/views/auto_complete_taxon_concepts_view/20230803181442.sql b/db/views/auto_complete_taxon_concepts_view/20230803181442.sql new file mode 100644 index 0000000000..65ba0f159b --- /dev/null +++ b/db/views/auto_complete_taxon_concepts_view/20230803181442.sql @@ -0,0 +1,199 @@ +WITH synonyms_segmented(taxon_concept_id, full_name, author_year, matched_taxon_concept_id, matched_name, matched_name_segment) AS ( + SELECT + atc.id, + atc.full_name, + atc.author_year, + tc.id, + tc.full_name, + UPPER(REGEXP_SPLIT_TO_TABLE(tc.full_name, ' ')) + FROM taxon_concepts tc + JOIN taxon_relationships tr + ON tr.other_taxon_concept_id = tc.id + JOIN taxon_relationship_types trt + ON trt.id = tr.taxon_relationship_type_id + AND trt.name = 'HAS_SYNONYM' + JOIN taxon_concepts atc + ON atc.id = tr.taxon_concept_id + WHERE tc.name_status = 'S' AND atc.name_status = 'A' +), scientific_names_segmented(taxon_concept_id, full_name, author_year, matched_taxon_concept_id, matched_name, matched_name_segment) AS ( + SELECT + id, + taxon_concepts.full_name, + taxon_concepts.author_year, + id, + taxon_concepts.full_name, + UPPER(REGEXP_SPLIT_TO_TABLE(full_name, ' ')) + FROM taxon_concepts +), unlisted_subspecies_segmented(taxon_concept_id, full_name, author_year, matched_taxon_concept_id, matched_name, matched_name_segment) AS ( + SELECT + parents.id, + parents.full_name, + parents.author_year, + taxon_concepts.id, + taxon_concepts.full_name, + UPPER(REGEXP_SPLIT_TO_TABLE(taxon_concepts.full_name, ' ')) + FROM taxon_concepts + JOIN ranks ON ranks.id = taxon_concepts.rank_id + AND ranks.name IN ('SUBSPECIES', 'VARIETY') + JOIN taxon_concepts parents + ON parents.id = taxon_concepts.parent_id + WHERE taxon_concepts.name_status NOT IN ('S', 'T', 'N') + AND parents.name_status = 'A' + + EXCEPT + + SELECT + parents.id, + parents.full_name, + parents.author_year, + taxon_concepts.id, + taxon_concepts.full_name, + UPPER(REGEXP_SPLIT_TO_TABLE(taxon_concepts.full_name, ' ')) + FROM taxon_concepts + JOIN ranks ON ranks.id = taxon_concepts.rank_id + AND ranks.name IN ('SUBSPECIES') -- VARIETY not here on purpose + JOIN taxon_concepts parents + ON parents.id = taxon_concepts.parent_id + JOIN taxonomies ON taxonomies.id = taxon_concepts.taxonomy_id + WHERE taxon_concepts.name_status NOT IN ('S', 'T', 'N') + AND parents.name_status = 'A' + AND CASE + WHEN taxonomies.name = 'CMS' + THEN (taxon_concepts.listing->'cms_historically_listed')::BOOLEAN + ELSE (taxon_concepts.listing->'cites_historically_listed')::BOOLEAN + OR (taxon_concepts.listing->'eu_historically_listed')::BOOLEAN + END +), taxon_common_names AS ( + SELECT + taxon_commons.*, + common_names.name + FROM taxon_commons + JOIN common_names + ON common_names.id = taxon_commons.common_name_id +), common_names_segmented(taxon_concept_id, full_name, author_year, matched_taxon_concept_id, matched_name, matched_name_segment) AS ( + SELECT + taxon_concept_id, + taxon_concepts.full_name, + taxon_concepts.author_year, + NULL::INT, + taxon_common_names.name, + UPPER(REGEXP_SPLIT_TO_TABLE(taxon_common_names.name, E'\\s|''')) + FROM taxon_common_names + JOIN taxon_concepts + ON taxon_common_names.taxon_concept_id = taxon_concepts.id +), taxon_common_names_dehyphenated AS ( + SELECT + taxon_concept_id, + taxon_concepts.full_name, + taxon_concepts.author_year, + NULL::INT, + taxon_common_names.name, + UPPER(REPLACE(taxon_common_names.name, '-', ' ')) + FROM taxon_common_names + JOIN taxon_concepts + ON taxon_common_names.taxon_concept_id = taxon_concepts.id + WHERE STRPOS(taxon_common_names.name, '-') > 0 +), common_names_segmented_dehyphenated AS ( + SELECT taxon_concept_id, full_name, author_year, matched_taxon_concept_id, matched_name, matched_name_segment + FROM common_names_segmented + UNION + SELECT taxon_concept_id, full_name, author_year, matched_taxon_concept_id, matched_name, REGEXP_SPLIT_TO_TABLE(matched_name_segment, E'-') + FROM common_names_segmented + WHERE STRPOS(matched_name_segment, '-') > 0 + UNION + SELECT * FROM taxon_common_names_dehyphenated +), all_names_segmented_cleaned AS ( + SELECT * FROM ( + SELECT taxon_concept_id, full_name, author_year, matched_taxon_concept_id, matched_name, + CASE + WHEN POSITION(matched_name_segment IN + UPPER(matched_name) + ) = 1 THEN UPPER(matched_name) + ELSE matched_name_segment + END, type_of_match + FROM ( + SELECT *, 'SELF' AS type_of_match + FROM scientific_names_segmented + UNION + SELECT *, 'SYNONYM' + FROM synonyms_segmented + UNION + SELECT *, 'SUBSPECIES' + FROM unlisted_subspecies_segmented + UNION + SELECT *, 'COMMON_NAME' + FROM common_names_segmented_dehyphenated + ) all_names_segmented + ) all_names_segmented_no_prefixes + WHERE LENGTH(matched_name_segment) >= 3 +), taxa_with_visibility_flags AS ( + SELECT taxon_concepts.id, + CASE + WHEN taxonomies.name = 'CITES_EU' THEN TRUE + ELSE FALSE + END AS taxonomy_is_cites_eu, + name_status, + ranks.name AS rank_name, + ranks.display_name_en AS rank_display_name_en, + ranks.display_name_es AS rank_display_name_es, + ranks.display_name_fr AS rank_display_name_fr, + ranks.taxonomic_position AS rank_order, + taxon_concepts.taxonomic_position, + CASE + WHEN + name_status = 'A' + AND ( + ranks.name != 'SUBSPECIES' + AND ranks.name != 'VARIETY' + OR taxonomies.name = 'CITES_EU' + AND ( + (listing->'cites_historically_listed')::BOOLEAN + OR (listing->'eu_historically_listed')::BOOLEAN + ) + OR taxonomies.name = 'CMS' + AND (listing->'cms_historically_listed')::BOOLEAN + ) + THEN TRUE + ELSE FALSE + END AS show_in_species_plus_ac, + CASE + WHEN + name_status = 'A' + AND ( + ranks.name != 'SUBSPECIES' + AND ranks.name != 'VARIETY' + OR (listing->'cites_show')::BOOLEAN + ) + THEN TRUE + ELSE FALSE + END AS show_in_checklist_ac, + CASE + WHEN + taxonomies.name = 'CITES_EU' + AND ARRAY['A', 'H', 'N']::VARCHAR[] && ARRAY[name_status] + THEN TRUE + ELSE FALSE + END AS show_in_trade_ac, + CASE + WHEN + taxonomies.name = 'CITES_EU' + AND ARRAY['A', 'H', 'N', 'T']::VARCHAR[] && ARRAY[name_status] + THEN TRUE + ELSE FALSE + END AS show_in_trade_internal_ac + FROM taxon_concepts + JOIN ranks ON ranks.id = rank_id + JOIN taxonomies ON taxonomies.id = taxon_concepts.taxonomy_id +) +SELECT + t1.*, + matched_name_segment AS name_for_matching, + matched_taxon_concept_id AS matched_id, + matched_name, + full_name, + author_year, + type_of_match +FROM taxa_with_visibility_flags t1 +JOIN all_names_segmented_cleaned t2 +ON t1.id = t2.taxon_concept_id +WHERE LENGTH(matched_name_segment) >= 3; diff --git a/lib/modules/import/rst/importer.rb b/lib/modules/import/rst/importer.rb index b9f0980457..21e39283af 100644 --- a/lib/modules/import/rst/importer.rb +++ b/lib/modules/import/rst/importer.rb @@ -2,6 +2,9 @@ module Import::Rst::Importer class << self def import(data) + # Array of CitesRstProcesses in current import we can use to + # destroy records no longer being returned from the RST API. + active_ids = [] data.map do |item| taxon_concept = map_taxon_concept(item) unless taxon_concept @@ -29,7 +32,10 @@ def import(data) start_date: item['startDate'], document: "https://rst.cites.org/public/case-details/#{item['id']}" ) + active_ids << rst_process.id end + + destroy_invalid_rst_processes(active_ids) end private @@ -49,5 +55,16 @@ def map_event(item) Rails.logger.info "Event #{item['meeting']['name']} for case #{item['id']} not found" unless event event end + + def destroy_invalid_rst_processes(active_ids) + CitesRstProcess.where.not(id: active_ids).find_each do |rst_process| + case_id = rst_process.case_id + if rst_process.destroy + Rails.logger.info "RST process with case_id #{case_id} destroyed" + else + Rails.logger.info "RST process with case_id #{case_id} could not be destroyed" + end + end + end end end diff --git a/lib/modules/search_param_sanitiser.rb b/lib/modules/search_param_sanitiser.rb index 0faf45074c..2a04e21415 100644 --- a/lib/modules/search_param_sanitiser.rb +++ b/lib/modules/search_param_sanitiser.rb @@ -6,7 +6,7 @@ def sanitise_string(s) end def sanitise_upcase_string(s) - s && s.strip.upcase + s && s.strip.mb_chars.upcase end def sanitise_symbol(s, default = nil) diff --git a/lib/modules/trade/download_data_retriever.rb b/lib/modules/trade/download_data_retriever.rb index f18a149980..3e7e414e6f 100644 --- a/lib/modules/trade/download_data_retriever.rb +++ b/lib/modules/trade/download_data_retriever.rb @@ -48,13 +48,22 @@ def self.search_download(params) SQL when 'species' appendix = params[:appendix] - <<-SQL - SELECT #{ATTRIBUTES.join(',')} - FROM non_compliant_shipments_view - WHERE year = #{year} - AND taxon_concept_id IN (#{id}) - AND appendix = '#{appendix}' - SQL + if appendix.present? + <<-SQL + SELECT #{ATTRIBUTES.join(',')} + FROM non_compliant_shipments_view + WHERE year = #{year} + AND taxon_concept_id IN (#{id}) + AND appendix = '#{appendix}' + SQL + else + <<-SQL + SELECT #{ATTRIBUTES.join(',')} + FROM non_compliant_shipments_view + WHERE year = #{year} + AND taxon_concept_id IN (#{id}) + SQL + end when 'commodity' <<-SQL SELECT #{ATTRIBUTES.join(',')} diff --git a/lib/modules/trade/grouping/base.rb b/lib/modules/trade/grouping/base.rb index 9943824abb..d105b9147e 100644 --- a/lib/modules/trade/grouping/base.rb +++ b/lib/modules/trade/grouping/base.rb @@ -35,6 +35,20 @@ def json_by_attribute(data, opts={}) raise NotImplementedError end + def read_taxonomy_conversion + conversion = {} + taxonomy = CSV.read(TAXONOMIC_GROUPING, {headers: true}) + taxonomy.each do |csv| + conversion[csv['group']] ||= [] + data = { + taxon_name: csv['taxon_name'], + rank: csv['taxonomic_level'] + } + conversion[csv['group']] << data + end + conversion + end + protected def shipments_table @@ -80,20 +94,6 @@ def limit private - def read_taxonomy_conversion - conversion = {} - taxonomy = CSV.read(TAXONOMIC_GROUPING, {headers: true}) - taxonomy.each do |csv| - conversion[csv['group']] ||= [] - data = { - taxon_name: csv['taxon_name'], - rank: csv['taxonomic_level'] - } - conversion[csv['group']] << data - end - conversion - end - def sanitise_group(group) return nil unless group attributes[group.to_sym] diff --git a/lib/modules/trade/grouping/compliance.rb b/lib/modules/trade/grouping/compliance.rb index d2a1a581e3..5ecef93c4f 100644 --- a/lib/modules/trade/grouping/compliance.rb +++ b/lib/modules/trade/grouping/compliance.rb @@ -271,7 +271,11 @@ def only_importer_countries(importers, keys, year) end def group_query - columns = @attributes.compact.uniq.join(',') + columns = if @attributes + @attributes.compact.uniq.join(',') + else + attributes.values.join(',') + end <<-SQL SELECT #{columns}, COUNT(*) AS cnt, 100.0*COUNT(*)/(SUM(COUNT(*)) OVER (PARTITION BY year)) AS percent FROM non_compliant_shipments_view