Search: specify package rank in generation; distinguish non-core Dart libs (dart-lang#3427)

Author: srawlins

lib/resources/docs.dart.js

@@ -170,7 +170,10 @@ abstract class GeneratorBackendBase implements GeneratorBackend {
   @override
   void generateSearchIndex(List<Indexable> indexedElements) {
     var json = generator_util.generateSearchIndexJson(
-        indexedElements, options.prettyIndexJson, options.packageOrder);
+      indexedElements,
+      packageOrder: options.packageOrder,
+      pretty: options.prettyIndexJson,
+    );
     if (!options.useBaseHref) {
       json = json.replaceAll(htmlBasePlaceholder, '');
     }

lib/resources/docs.dart.js.map

@@ -35,19 +35,24 @@ String generateCategoryJson(Iterable<Categorization> categories, bool pretty) {
   return encoder.convert(indexItems.toList(growable: false));
 }
 
-String generateSearchIndexJson(Iterable<Indexable> indexedElements, bool pretty,
-    List<String> packageOrder) {
+/// Generates the text of the search index file (`index.json`) containing
+/// [indexedItems] and [packageOrder].
+///
+/// Passing `pretty: true` will use a [JsonEncoder] with a single-space indent.
+String generateSearchIndexJson(Iterable<Indexable> indexedElements,
+    {required List<String> packageOrder, required bool pretty}) {
   final indexItems = [
-    {packageOrderKey: packageOrder},
     for (final indexable
         in indexedElements.sorted(_compareElementRepresentations))
       {
         'name': indexable.name,
         'qualifiedName': indexable.fullyQualifiedName,
         'href': indexable.href,
         'kind': indexable.kind.index,
+        // TODO(srawlins): Only include this for [Inheritable] items.
         'overriddenDepth': indexable.overriddenDepth,
-        if (indexable is ModelElement) 'packageName': indexable.package.name,
+        if (indexable is ModelElement)
+          'packageRank': _packageRank(packageOrder, indexable),
         if (indexable is ModelElement)
           'desc': _removeHtmlTags(indexable.oneLineDoc),
         if (indexable is EnclosedElement)
@@ -65,8 +70,44 @@ String generateSearchIndexJson(Iterable<Indexable> indexedElements, bool pretty,
   return encoder.convert(indexItems);
 }
 
-/// The key used in the `index.json` file used to specify the package order.
-const packageOrderKey = '__PACKAGE_ORDER__';
+/// The "package rank" of [element], given a [packageOrder].
+///
+/// Briefly, this is 10 times the element's package's position in the
+/// [packageOrder], or 10 times the length of [packageOrder] if the element's
+/// package is not listed in [packageOrder].
+int _packageRank(List<String> packageOrder, ModelElement element) {
+  if (packageOrder.isEmpty) return 0;
+  var packageName = element.package.name;
+  var index = packageOrder.indexOf(packageName);
+  if (index == -1) return packageOrder.length * 10;
+  if (packageName == 'Dart' &&
+      !_dartCoreLibraries.contains(element.library.name)) {
+    // Non-"core" Dart SDK libraries should be ranked slightly lower than "core"
+    // Dart SDK libraries. The "core" Dart SDK libraries are the ones labeled as
+    // such at <https://api.dart.dev>, which can be used in both VM and Web
+    // environments.
+    // Note we choose integers throughout this function (as opposed to adding
+    // 0.5 here) in order to facilitate a proper [Comparable] comparison.
+    return index * 10 + 5;
+  }
+  return index * 10;
+}
+
+/// The set of "core" Dart libraries, used to rank contained items above items
+/// declared elsewhere in the Dart SDK.
+const _dartCoreLibraries = {
+  'dart:async',
+  'dart:collection',
+  'dart:convert',
+  'dart:core',
+  'dart:developer',
+  'dart:math',
+  'dart:typed_data',
+
+  // Plus the two core Flutter engine libraries.
+  'dart:ui',
+  'dart:web_ui',
+};
 
 String _removeHtmlTags(String? input) =>
     input?.replaceAll(_htmlTagPattern, '') ?? '';

lib/src/generator/generator_backend.dart

@@ -4,7 +4,6 @@
 
 import 'dart:convert';
 
-import 'package:dartdoc/src/generator/generator_utils.dart';
 import 'package:dartdoc/src/model/indexable.dart';
 import 'package:meta/meta.dart';
 
@@ -17,30 +16,14 @@ enum _MatchPosition {
 }
 
 class Index {
-  final List<String> packageOrder;
   final List<IndexItem> index;
 
   @visibleForTesting
-  Index(this.packageOrder, this.index);
+  Index(this.index);
 
   factory Index.fromJson(String text) {
     var jsonIndex = (jsonDecode(text) as List).cast<Map<String, dynamic>>();
-    var indexList = <IndexItem>[];
-    var packageOrder = <String>[];
-    for (var entry in jsonIndex) {
-      if (entry.containsKey(packageOrderKey)) {
-        packageOrder.addAll((entry[packageOrderKey] as List).cast<String>());
-      } else {
-        indexList.add(IndexItem.fromMap(entry));
-      }
-    }
-    return Index(packageOrder, indexList);
-  }
-
-  int packageOrderPosition(String packageName) {
-    if (packageOrder.isEmpty) return 0;
-    var index = packageOrder.indexOf(packageName);
-    return index == -1 ? packageOrder.length : index;
+    return Index(jsonIndex.map(IndexItem.fromMap).toList());
   }
 
   List<IndexItem> find(String rawQuery) {
@@ -84,8 +67,7 @@ class Index {
       }
 
       // Prefer packages higher in the package order.
-      comparison = packageOrderPosition(a.item.packageName) -
-          packageOrderPosition(b.item.packageName);
+      comparison = a.item.packageRank - b.item.packageRank;
       if (comparison != 0) {
         return comparison;
       }
@@ -113,11 +95,7 @@ class Index {
 class IndexItem {
   final String name;
   final String qualifiedName;
-
-  // TODO(srawlins): Store the index of the package in package order instead of
-  // this String. The Strings bloat the `index.json` file and keeping duplicate
-  // parsed Strings in memory is expensive.
-  final String packageName;
+  final int packageRank;
   final Kind kind;
   final String? href;
   final int overriddenDepth;
@@ -127,7 +105,7 @@ class IndexItem {
   IndexItem._({
     required this.name,
     required this.qualifiedName,
-    required this.packageName,
+    required this.packageRank,
     required this.kind,
     required this.desc,
     required this.href,
@@ -144,14 +122,11 @@ class IndexItem {
   //   "href":"dartdoc/Dartdoc-class.html",
   //   "kind":1,
   //   "overriddenDepth":0,
-  //   "packageName":"dartdoc"
+  //   "packageRank":0
   //   ["enclosedBy":{"name":"dartdoc","kind":2}]
   // }
   // ```
   factory IndexItem.fromMap(Map<String, dynamic> data) {
-    // Note that this map also contains 'packageName', but we're not currently
-    // using that info.
-
     EnclosedBy? enclosedBy;
     if (data['enclosedBy'] != null) {
       final map = data['enclosedBy'] as Map<String, dynamic>;
@@ -164,7 +139,7 @@ class IndexItem {
     return IndexItem._(
       name: data['name'],
       qualifiedName: data['qualifiedName'],
-      packageName: data['packageName'],
+      packageRank: data['packageRank'] as int,
       href: data['href'],
       kind: Kind.values[data['kind'] as int],
       overriddenDepth: (data['overriddenDepth'] as int?) ?? 0,

lib/src/generator/generator_utils.dart

@@ -40,6 +40,8 @@ abstract class DartdocTestBase {
 
   List<String> get experiments => [];
 
+  bool get skipUnreachableSdkLibraries => true;
+
   @mustCallSuper
   Future<void> setUp() async {
     packageMetaProvider = testPackageMetaProvider;
@@ -83,12 +85,13 @@ analyzer:
       packageMetaProvider,
       packageConfigProvider,
       additionalArguments: additionalArguments,
+      skipUnreachableSdkLibraries: skipUnreachableSdkLibraries,
     );
   }
 
   /// Creates a single library named [libraryName], with optional preamble
-  /// [libraryPreamble].   Optionally, pass a [FileGenerator] to create
-  /// extra files in the package such as `dartdoc_options.yaml`.
+  /// [libraryPreamble].  Optionally, pass [extraFiles] such as
+  /// `dartdoc_options.yaml`.
   Future<Library> bootPackageWithLibrary(String libraryContent,
       {String libraryPreamble = '',
       Iterable<d.Descriptor> extraFiles = const [],

lib/src/search.dart

@@ -0,0 +1,184 @@
+// Copyright (c) 2023, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:convert';
+
+import 'package:dartdoc/src/generator/generator_utils.dart';
+import 'package:dartdoc/src/model/model.dart';
+import 'package:test/test.dart';
+import 'package:test_reflective_loader/test_reflective_loader.dart';
+
+import 'dartdoc_test_base.dart';
+import 'src/utils.dart';
+
+void main() {
+  defineReflectiveSuite(() {
+    if (recordsAllowed) {
+      defineReflectiveTests(SearchIndexTest);
+    }
+  });
+}
+
+@reflectiveTest
+class SearchIndexTest extends DartdocTestBase {
+  @override
+  String get libraryName => 'index_json';
+
+  /// We need the "unreachable" SDK libraries in order to include the non-"core"
+  /// ones like 'dart:io'.
+  @override
+  bool get skipUnreachableSdkLibraries => false;
+
+  /// Returns the JSON-decoded text of the search index which is generated with
+  /// the elements from [libraryContent].
+  ///
+  /// Specify `actAsFlutter: true` to auto-include dependencies, and to specify
+  /// a package order with "flutter" and "Dart".
+  Future<List<Map<String, dynamic>>> jsonIndexForPackageWithLibrary(
+    String libraryContent, {
+    bool actAsFlutter = false,
+  }) async {
+    var library = await bootPackageWithLibrary(
+      libraryContent,
+      additionalArguments:
+          actAsFlutter ? const ['--auto-include-dependencies'] : const [],
+    );
+    var libraries = [
+      library,
+      if (actAsFlutter)
+        ...library.packageGraph.libraries
+            .where((l) => l.name.startsWith('dart:')),
+    ];
+    var text = generateSearchIndexJson(
+      [
+        // For each library, gather the elements for the library, it's members,
+        // and the members of its container members.
+        for (var library in libraries) ...[
+          library,
+          ...library.allModelElements,
+          for (var element in library.allModelElements.whereType<Container>())
+            ...element.allModelElements,
+        ],
+      ],
+      packageOrder: actAsFlutter ? const ['flutter', 'Dart'] : const [],
+      pretty: false,
+    );
+    return (jsonDecode(text) as List).cast<Map<String, dynamic>>();
+  }
+
+  void test_class() async {
+    var jsonIndex = await jsonIndexForPackageWithLibrary('''
+/// A class.
+class C {}
+''');
+    var classItem = jsonIndex.named('index_json.C');
+
+    expect(classItem['kind'], equals(Kind.class_.index));
+    expect(classItem['overriddenDepth'], equals(0));
+    expect(classItem['desc'], equals('A class.'));
+    expect(
+      classItem['enclosedBy'],
+      equals({
+        'name': 'index_json',
+        'kind': Kind.library.index,
+        'href':
+            '%%__HTMLBASE_dartdoc_internal__%%index_json/index_json-library.html',
+      }),
+    );
+  }
+
+  void test_library() async {
+    var jsonIndex = await jsonIndexForPackageWithLibrary('''
+/// A library.
+library;
+''');
+    var libraryItem = jsonIndex.named('index_json');
+
+    expect(libraryItem['kind'], equals(Kind.library.index));
+    expect(libraryItem['overriddenDepth'], equals(0));
+    // TODO(srawlins): Should not be blank.
+    expect(libraryItem['desc'], equals(''));
+  }
+
+  void test_method() async {
+    var jsonIndex = await jsonIndexForPackageWithLibrary('''
+class C {
+  /// A method.
+  void m() {}
+}
+''');
+    var methodItem = jsonIndex.named('index_json.C.m');
+
+    expect(methodItem['kind'], equals(Kind.method.index));
+    expect(methodItem['overriddenDepth'], equals(0));
+    expect(methodItem['desc'], equals('A method.'));
+    expect(
+      methodItem['enclosedBy'],
+      equals({
+        'name': 'C',
+        'kind': Kind.class_.index,
+        'href': '%%__HTMLBASE_dartdoc_internal__%%index_json/C-class.html',
+      }),
+    );
+  }
+
+  void test_overriddenMethod() async {
+    var jsonIndex = await jsonIndexForPackageWithLibrary('''
+class C {
+  void m() {}
+}
+class D extends C {
+  /// A method.
+  @override
+  void m() {}
+}
+''');
+    var methodItem = jsonIndex.named('index_json.D.m');
+
+    expect(methodItem['kind'], equals(Kind.method.index));
+    expect(methodItem['overriddenDepth'], equals(1));
+    expect(methodItem['desc'], equals('A method.'));
+    expect(
+      methodItem['enclosedBy'],
+      equals({
+        'name': 'D',
+        'kind': Kind.class_.index,
+        'href': '%%__HTMLBASE_dartdoc_internal__%%index_json/D-class.html',
+      }),
+    );
+  }
+
+  void test_sdkPackageRank_core() async {
+    var jsonIndex = await jsonIndexForPackageWithLibrary(
+      actAsFlutter: true,
+      '''
+/// A library.
+library;
+''',
+    );
+    var classItem = jsonIndex.named('dart:core.List');
+
+    expect(classItem['kind'], equals(Kind.class_.index));
+    expect(classItem['packageRank'], equals(10));
+  }
+
+  void test_sdkPackageRank_nonCore() async {
+    var jsonIndex = await jsonIndexForPackageWithLibrary(
+      actAsFlutter: true,
+      '''
+/// A library.
+library;
+''',
+    );
+    var classItem = jsonIndex.named('dart:io.File');
+
+    expect(classItem['kind'], equals(Kind.class_.index));
+    expect(classItem['packageRank'], equals(15));
+  }
+}
+
+extension on List<Map<String, dynamic>> {
+  Map<String, dynamic> named(String qualifiedName) =>
+      firstWhere((e) => e['qualifiedName'] == qualifiedName);
+}