diff --git a/tests/std/tests/P1502R1_standard_library_header_units/custom_format.py b/tests/std/tests/P1502R1_standard_library_header_units/custom_format.py index 7008dab9de..4cf472b9e0 100644 --- a/tests/std/tests/P1502R1_standard_library_header_units/custom_format.py +++ b/tests/std/tests/P1502R1_standard_library_header_units/custom_format.py @@ -13,93 +13,9 @@ noisyProgress = False -# P1502R1_standard_library_header_units/test.cpp cites the definition of "importable C++ library headers". -def getImportableCxxLibraryHeaders(): - return [ - 'algorithm', - 'any', - 'array', - 'atomic', - 'barrier', - 'bit', - 'bitset', - 'charconv', - 'chrono', - 'codecvt', - 'compare', - 'complex', - 'concepts', - 'condition_variable', - 'coroutine', - 'deque', - 'exception', - 'execution', - 'filesystem', - 'format', - 'forward_list', - 'fstream', - 'functional', - 'future', - 'initializer_list', - 'iomanip', - 'ios', - 'iosfwd', - 'iostream', - 'istream', - 'iterator', - 'latch', - 'limits', - 'list', - 'locale', - 'map', - 'memory', - 'memory_resource', - 'mutex', - 'new', - 'numbers', - 'numeric', - 'optional', - 'ostream', - 'queue', - 'random', - 'ranges', - 'ratio', - 'regex', - 'scoped_allocator', - 'semaphore', - 'set', - 'shared_mutex', - 'source_location', - 'span', - 'spanstream', - 'sstream', - 'stack', - 'stdexcept', - 'stop_token', - 'streambuf', - 'string', - 'string_view', - 'strstream', - 'syncstream', - 'system_error', - 'thread', - 'tuple', - 'type_traits', - 'typeindex', - 'typeinfo', - 'unordered_map', - 'unordered_set', - 'utility', - 'valarray', - 'variant', - 'vector', - 'version', - ] - - def loadJsonWithComments(filename): # This is far from a general-purpose solution (it doesn't attempt to handle block comments like /**/ - # and comments appearing within strings like "cats // dogs"), but it's sufficient for header-units.json. + # and comments appearing within strings like "cats // dogs"), but it's sufficient for our purposes. comment = re.compile('//.*') with open(filename) as file: replacedLines = [re.sub(comment, '', line) for line in file] @@ -111,7 +27,14 @@ def getAllHeaders(headerUnitsJsonFilename): # We want to build everything that's mentioned in header-units.json, plus all of the # headers that were commented out for providing macros that control header inclusion. - return sorted(set(buildAsHeaderUnits + ['version', 'yvals.h', 'yvals_core.h'])) + return buildAsHeaderUnits + ['version', 'yvals.h', 'yvals_core.h'] + + +def getImportableCxxLibraryHeaders(sourcePath): + # This JSON With Comments file is shared between Python and Perl, + # reducing the number of things we need to update when adding new Standard headers. + jsonFilename = os.path.join(os.path.dirname(sourcePath), 'importable_cxx_library_headers.jsonc') + return loadJsonWithComments(jsonFilename) class CustomTestFormat(STLTestFormat): @@ -188,7 +111,7 @@ def getBuildSteps(self, test, litConfig, shared): for hdr in readyToBuild: consumeBuiltHeaderUnits += ['/headerUnit:angle', f'{hdr}={hdr}.ifc'] else: # Build independent header units: - stlHeaders = getImportableCxxLibraryHeaders() + stlHeaders = getImportableCxxLibraryHeaders(sourcePath) exportHeaderOptions = ['/exportHeader', '/headerName:angle', '/Fo', '/MP'] for hdr in stlHeaders: consumeBuiltHeaderUnits += ['/headerUnit:angle', f'{hdr}={hdr}.ifc'] diff --git a/tests/std/tests/P1502R1_standard_library_header_units/custombuild.pl b/tests/std/tests/P1502R1_standard_library_header_units/custombuild.pl index 4e23980600..8ffb446266 100644 --- a/tests/std/tests/P1502R1_standard_library_header_units/custombuild.pl +++ b/tests/std/tests/P1502R1_standard_library_header_units/custombuild.pl @@ -1,105 +1,166 @@ # Copyright (c) Microsoft Corporation. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +use strict; +use warnings; + +use JSON::PP; use Run; +sub readFile +{ + my $filename = $_[0]; + open(my $handle, "<", $filename) or die("Couldn't open $filename: $!"); + read($handle, my $string, -s $handle); + return $string; +} + +sub loadJson +{ + my $filename = $_[0]; + my $jsonStr = readFile($filename); + return JSON::PP->new->utf8->decode($jsonStr); +} + +sub loadJsonWithComments +{ + my $filename = $_[0]; + my $jsonStr = readFile($filename); + return JSON::PP->new->relaxed->utf8->decode($jsonStr); +} + +sub getAllHeaders +{ + my $headerUnitsJsonFilename = $_[0]; + my $jsonObject = loadJsonWithComments($headerUnitsJsonFilename); + my @buildAsHeaderUnits = @{$jsonObject->{"BuildAsHeaderUnits"}}; + # We want to build everything that's mentioned in header-units.json, plus all of the + # headers that were commented out for providing macros that control header inclusion. + push(@buildAsHeaderUnits, "version", "yvals.h", "yvals_core.h"); + return @buildAsHeaderUnits; +} + +sub getImportableCxxLibraryHeaders() +{ + # This JSON With Comments file is shared between Python and Perl, + # reducing the number of things we need to update when adding new Standard headers. + my $jsonObject = loadJsonWithComments("importable_cxx_library_headers.jsonc"); + return @{$jsonObject}; +} + +sub arrayDifference +{ + # (Perl) Takes two arrays by reference. Returns everything in the first array (minuend) that doesn't appear + # in the second array (subtrahend). Doesn't require the arrays to be sorted, so the complexity is O(M * N). + my @minuend = @{$_[0]}; + my @subtrahend = @{$_[1]}; + + my @result = (); + foreach my $elem (@minuend) { + if (!grep($elem eq $_, @subtrahend)) { + push(@result, $elem); + } + } + return @result; +} + sub CustomBuildHook() { my $cwd = Run::GetCWDName(); - my @stl_headers = ( - "algorithm", - "any", - "array", - "atomic", - "barrier", - "bit", - "bitset", - "charconv", - "chrono", - "codecvt", - "compare", - "complex", - "concepts", - "condition_variable", - "coroutine", - "deque", - "exception", - "execution", - "filesystem", - "format", - "forward_list", - "fstream", - "functional", - "future", - "initializer_list", - "iomanip", - "ios", - "iosfwd", - "iostream", - "istream", - "iterator", - "latch", - "limits", - "list", - "locale", - "map", - "memory", - "memory_resource", - "mutex", - "new", - "numbers", - "numeric", - "optional", - "ostream", - "queue", - "random", - "ranges", - "ratio", - "regex", - "scoped_allocator", - "semaphore", - "set", - "shared_mutex", - "source_location", - "span", - "spanstream", - "sstream", - "stack", - "stdexcept", - "stop_token", - "streambuf", - "string", - "string_view", - "strstream", - "syncstream", - "system_error", - "thread", - "tuple", - "type_traits", - "typeindex", - "typeinfo", - "unordered_map", - "unordered_set", - "utility", - "valarray", - "variant", - "vector", - "version", - ); - - my $export_header_options = "/exportHeader /headerName:angle /Fo /MP"; - my $header_unit_options = ""; - - foreach (@stl_headers) { - $export_header_options .= " $_"; - - $header_unit_options .= " /headerUnit:angle"; - $header_unit_options .= " $_=$_.ifc"; - $header_unit_options .= " $_.obj"; + # This is a list of compiler options to consume the header units that we've built so far. + my @consumeBuiltHeaderUnits = (); + + # Output files: + my @objFilenames = (); + + if ($ENV{PM_CL} =~ m) { # Build deduplicated header units: + # Compiler options, common to both scanning dependencies and building header units. + my @clOptions = ("/exportHeader", "/headerName:angle", "/translateInclude", "/Fo", "/MP"); + + # Store the list of headers to build. + my $stlIncludeDir = $ENV{STL_INCLUDE_DIR}; + my @allHeaders = getAllHeaders("$stlIncludeDir\\header-units.json"); + + # Generate JSON files that record how these headers depend on one another. + Run::ExecuteCL(join(" ", @clOptions, "/scanDependencies", ".\\", @allHeaders)); + + # The JSON files also record what object files will be produced. + # IFC filenames and OBJ filenames follow different patterns. For example: + # produces filesystem.ifc and filesystem.obj + # produces xbit_ops.h.ifc and xbit_ops.obj + # We can easily synthesize IFC filenames, but it's easier to get the OBJ filenames from the JSON files. + + # This dictionary powers the topological sort. + # Key: Header name, e.g. 'bitset'. + # Value: List of dependencies that remain to be built, e.g. ['iosfwd', 'limits', 'xstring']. + my %remainingDependencies; + + # Read the JSON files, storing the results in objFilenames and remainingDependencies. + foreach my $hdr (@allHeaders) { + my $jsonObject = loadJson("$hdr.module.json"); + push(@objFilenames, $jsonObject->{"rules"}[0]{"primary-output"}); + + my @dep = (); + foreach my $req (@{$jsonObject->{"rules"}[0]{"requires"}}) { + push(@dep, $req->{"logical-name"}); + } + $remainingDependencies{$hdr} = \@dep; + } + + # Build header units in topologically sorted order. + while (%remainingDependencies) { + # When a header has no remaining dependencies, it is ready to be built. + my @readyToBuild = (); + foreach my $hdr (keys(%remainingDependencies)) { + my @dep = @{$remainingDependencies{$hdr}}; + if (!@dep) { + push(@readyToBuild, $hdr); + } + } + + # Each layer can be built in parallel. + Run::ExecuteCL(join(" ", @clOptions, @consumeBuiltHeaderUnits, @readyToBuild)); + + # Update remainingDependencies by doing two things. + + # (Perl) First, eliminate headers that we just built. + foreach my $hdr (@readyToBuild) { + delete $remainingDependencies{$hdr}; + } + + # hdr, dep is the current key-value pair. + foreach my $hdr (keys(%remainingDependencies)) { + my @dep = @{$remainingDependencies{$hdr}}; + + # Second, filter dep, eliminating anything that appears in readyToBuild. (If we're left with + # an empty list, then hdr will be ready to build in the next iteration.) + my @filtered = arrayDifference(\@dep, \@readyToBuild); + + $remainingDependencies{$hdr} = \@filtered; + } + + # Add compiler options to consume the header units that we just built. + foreach my $hdr (@readyToBuild) { + push(@consumeBuiltHeaderUnits, "/headerUnit:angle", "$hdr=$hdr.ifc"); + } + } + } else { # Build independent header units: + my @stlHeaders = getImportableCxxLibraryHeaders(); + my @exportHeaderOptions = ("/exportHeader", "/headerName:angle", "/Fo", "/MP"); + + foreach my $hdr (@stlHeaders) { + push(@consumeBuiltHeaderUnits, "/headerUnit:angle", "$hdr=$hdr.ifc"); + push(@objFilenames, "$hdr.obj"); + } + + Run::ExecuteCL(join(" ", @exportHeaderOptions, @stlHeaders)); } - Run::ExecuteCL("$export_header_options"); - Run::ExecuteCL("test.cpp /Fe$cwd.exe $header_unit_options"); + # For convenience, create a library file containing all of the object files that were produced. + my $libFilename = "stl_header_units.lib"; + Run::ExecuteCommand(join(" ", "lib.exe", "/nologo", "/out:$libFilename", @objFilenames)); + + Run::ExecuteCL(join(" ", "test.cpp", "/Fe$cwd.exe", @consumeBuiltHeaderUnits, $libFilename)); } 1 diff --git a/tests/std/tests/P1502R1_standard_library_header_units/importable_cxx_library_headers.jsonc b/tests/std/tests/P1502R1_standard_library_header_units/importable_cxx_library_headers.jsonc new file mode 100644 index 0000000000..ea9dda8147 --- /dev/null +++ b/tests/std/tests/P1502R1_standard_library_header_units/importable_cxx_library_headers.jsonc @@ -0,0 +1,84 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +// P1502R1_standard_library_header_units/test.cpp cites the definition of "importable C++ library headers". +[ + "algorithm", + "any", + "array", + "atomic", + "barrier", + "bit", + "bitset", + "charconv", + "chrono", + "codecvt", + "compare", + "complex", + "concepts", + "condition_variable", + "coroutine", + "deque", + "exception", + "execution", + "filesystem", + "format", + "forward_list", + "fstream", + "functional", + "future", + "initializer_list", + "iomanip", + "ios", + "iosfwd", + "iostream", + "istream", + "iterator", + "latch", + "limits", + "list", + "locale", + "map", + "memory", + "memory_resource", + "mutex", + "new", + "numbers", + "numeric", + "optional", + "ostream", + "queue", + "random", + "ranges", + "ratio", + "regex", + "scoped_allocator", + "semaphore", + "set", + "shared_mutex", + "source_location", + "span", + "spanstream", + "sstream", + "stack", + "stdexcept", + "stop_token", + "streambuf", + "string", + "string_view", + "strstream", + "syncstream", + "system_error", + "thread", + "tuple", + "type_traits", + "typeindex", + "typeinfo", + "unordered_map", + "unordered_set", + "utility", + "valarray", + "variant", + "vector", + "version" +] diff --git a/tests/std/tests/P1502R1_standard_library_header_units/test.cpp b/tests/std/tests/P1502R1_standard_library_header_units/test.cpp index 118a06f90d..3cfe0751a4 100644 --- a/tests/std/tests/P1502R1_standard_library_header_units/test.cpp +++ b/tests/std/tests/P1502R1_standard_library_header_units/test.cpp @@ -285,7 +285,7 @@ int main() { assert(!ep); } -#ifndef TEST_TOPO_SORT // TRANSITION, VSO-1471382 (error C2672: 'count_if': no matching overloaded function found) +#if !defined(TEST_TOPO_SORT) || defined(_MSVC_INTERNAL_TESTING) // TRANSITION, VSO-1471382 fixed in VS 2022 17.2p2 { puts("Testing ."); constexpr int arr[]{11, 0, 22, 0, 33, 0, 44, 0, 55}; @@ -321,7 +321,7 @@ int main() { assert(!f.is_open()); } -#ifndef TEST_TOPO_SORT // TRANSITION, VSO-1471374 (fatal error C1116: unrecoverable error importing module) +#if !defined(TEST_TOPO_SORT) || defined(_MSVC_INTERNAL_TESTING) // TRANSITION, VSO-1471374 fixed in VS 2022 17.2p2 { puts("Testing ."); function f{multiplies{}}; @@ -865,7 +865,7 @@ int main() { assert(this_thread::get_id() != thread::id{}); } -#ifndef TEST_TOPO_SORT // TRANSITION, VSO-1471374 (fatal error C1116: unrecoverable error importing module) +#if !defined(TEST_TOPO_SORT) || defined(_MSVC_INTERNAL_TESTING) // TRANSITION, VSO-1471374 fixed in VS 2022 17.2p2 { puts("Testing ."); constexpr tuple t{1729, 'c', 1.25};