From e1d0b38d5debd0f79f58708ef6468a0eba3a7594 Mon Sep 17 00:00:00 2001 From: Onsi Fakhouri Date: Thu, 5 Oct 2023 09:23:20 -0600 Subject: [PATCH] Add PreviewSpecs() to enable programmatic preview access to the suite report --- core_dsl.go | 91 +++++++++++++------ docs/index.md | 26 +++++- dsl/core/core_dsl.go | 1 + .../preview_fixture_suite_test.go | 41 +++++++++ integration/preview_test.go | 45 +++++++++ internal/suite.go | 10 ++ types/errors.go | 15 +++ 7 files changed, 202 insertions(+), 27 deletions(-) create mode 100644 integration/_fixtures/preview_fixture/preview_fixture_suite_test.go create mode 100644 integration/preview_test.go diff --git a/core_dsl.go b/core_dsl.go index a244bdc18..a6a3a8dcf 100644 --- a/core_dsl.go +++ b/core_dsl.go @@ -38,7 +38,7 @@ var flagSet types.GinkgoFlagSet var deprecationTracker = types.NewDeprecationTracker() var suiteConfig = types.NewDefaultSuiteConfig() var reporterConfig = types.NewDefaultReporterConfig() -var suiteDidRun = false +var suiteDidRun, suiteDidPreview = false, false var outputInterceptor internal.OutputInterceptor var client parallel_support.Client @@ -247,32 +247,12 @@ func RunSpecs(t GinkgoTestingT, description string, args ...interface{}) bool { if suiteDidRun { exitIfErr(types.GinkgoErrors.RerunningSuite()) } - suiteDidRun = true - - suiteLabels := Labels{} - configErrors := []error{} - for _, arg := range args { - switch arg := arg.(type) { - case types.SuiteConfig: - suiteConfig = arg - case types.ReporterConfig: - reporterConfig = arg - case Labels: - suiteLabels = append(suiteLabels, arg...) - default: - configErrors = append(configErrors, types.GinkgoErrors.UnknownTypePassedToRunSpecs(arg)) - } + if suiteDidPreview { + exitIfErr(types.GinkgoErrors.RunAndPreviewSuite()) } - exitIfErrors(configErrors) + suiteDidRun = true - configErrors = types.VetConfig(flagSet, suiteConfig, reporterConfig) - if len(configErrors) > 0 { - fmt.Fprintf(formatter.ColorableStdErr, formatter.F("{{red}}Ginkgo detected configuration issues:{{/}}\n")) - for _, err := range configErrors { - fmt.Fprintf(formatter.ColorableStdErr, err.Error()) - } - os.Exit(1) - } + suiteLabels := extractSuiteConfiguration(args) var reporter reporters.Reporter if suiteConfig.ParallelTotal == 1 { @@ -310,7 +290,6 @@ func RunSpecs(t GinkgoTestingT, description string, args ...interface{}) bool { err := global.Suite.BuildTree() exitIfErr(err) - suitePath, err := os.Getwd() exitIfErr(err) suitePath, err = filepath.Abs(suitePath) @@ -335,6 +314,66 @@ func RunSpecs(t GinkgoTestingT, description string, args ...interface{}) bool { return passed } +func extractSuiteConfiguration(args []interface{}) Labels { + suiteLabels := Labels{} + configErrors := []error{} + for _, arg := range args { + switch arg := arg.(type) { + case types.SuiteConfig: + suiteConfig = arg + case types.ReporterConfig: + reporterConfig = arg + case Labels: + suiteLabels = append(suiteLabels, arg...) + default: + configErrors = append(configErrors, types.GinkgoErrors.UnknownTypePassedToRunSpecs(arg)) + } + } + exitIfErrors(configErrors) + + configErrors = types.VetConfig(flagSet, suiteConfig, reporterConfig) + if len(configErrors) > 0 { + fmt.Fprintf(formatter.ColorableStdErr, formatter.F("{{red}}Ginkgo detected configuration issues:{{/}}\n")) + for _, err := range configErrors { + fmt.Fprintf(formatter.ColorableStdErr, err.Error()) + } + os.Exit(1) + } + + return suiteLabels +} + +/* +PreviewSpecs walks the testing tree and produces a report without actually invoking the specs. +See http://onsi.github.io/ginkgo/#previewing-specs for more information. +*/ +func PreviewSpecs(description string, args ...any) Report { + if suiteDidRun { + exitIfErr(types.GinkgoErrors.RunAndPreviewSuite()) + } + + suiteLabels := extractSuiteConfiguration(args) + if suiteConfig.ParallelTotal != 1 { + exitIfErr(types.GinkgoErrors.PreviewInParallelConfiguration()) + } + suiteConfig.DryRun = true + reporter := reporters.NoopReporter{} + outputInterceptor = internal.NoopOutputInterceptor{} + client = nil + writer := GinkgoWriter.(*internal.Writer) + + err := global.Suite.BuildTree() + exitIfErr(err) + suitePath, err := os.Getwd() + exitIfErr(err) + suitePath, err = filepath.Abs(suitePath) + exitIfErr(err) + + global.Suite.Run(description, suiteLabels, suitePath, global.Failer, reporter, writer, outputInterceptor, interrupt_handler.NewInterruptHandler(client), client, internal.RegisterForProgressSignal, suiteConfig) + + return global.Suite.GetPreviewReport() +} + /* Skip instructs Ginkgo to skip the current spec diff --git a/docs/index.md b/docs/index.md index 79da2e9e7..fd440a4a9 100644 --- a/docs/index.md +++ b/docs/index.md @@ -3183,6 +3183,30 @@ A single interrupt (e.g. `SIGINT`/`SIGTERM`) interrupts the current running node If you want to get information about what is currently running in a suite _without_ interrupting it, check out the [Getting Visibility Into Long-Running Specs](#getting-visibility-into-long-running-specs) section above. +### Previewing Specs + +Ginkgo provides a few different mechansisms for previewing and analyzing the specs defined in a suite. You can use the [`outline`](#creating-an-outline-of-specs) cli command to get a machine-readable list of specs defined in the suite. Outline parses the Go AST tree of the suite to determine the specs and therefore does not require the suite to be compiled. This comes with a limitation, however: outline does not offer insight into which specs will run for a given set of filters and it cannot handle dynamically generated specs (example specs generated by a `for` loop). + +For a more complete preview you can run `ginkgo --dry-run -v`. This compiles the spec, builds the spec tree, and then walks the tree printing out spec information using Ginkgo's default output as it goes. This allows you to see which specs will run for a given set of filters and also allows you to see dynamically generated specs. Note that you cannot use `--dry-run` with `-p` or `-procs`: you must run in series. + +If, you need finer-grained control over previews you can use `PreviewSpecs` in your suite in lieu of `RunSpecs`. `PreviewSpecs` behaves like `--dry-run` in that it will compile the suite, build the spec tree, and then walk the tree while honoring any filter and randomization flags. However `PreviewSpecs` generates and returns a full [`Report` object](#reporting-nodes---reportbeforesuite-and-reportaftersuite) that can be manipulated and inspected as needed. Specs that will be run will have `State = SpecStatePassed` and specs that will be skipped will have `SpecStateSkipped`. + +Currently you must run in series to invoke `PreviewSpecs` and you cannot run both `PreviewSpecs` and `RunSpecs` in the same suite. If you are opting into `PreviewSpecs` in lieu of `--dry-run` one suggested pattern is to key off of the `--dry-run` configuration to run `PreviewSpecs` instead of `RunSpecs`: + +```go +func TestMySuite(t *testing.T) { + config, _ := GinkgoConfiguration() + if config.DryRun { + report := PreviewSpecs("My Suite", Label("suite-label")) + //...do things with report. e.g. reporters.GenerateJUnitReport(report, "./preview.xml") + } else { + RunSpecs(t, "My Suite", Label("suite-label")) + } +} +``` + +Note that since `RunSuite` accepts a description string and decorators that can influence the spec tree, you'll want to use the same arguments with `PreviewSpecs`. + ### Running Multiple Suites So far we've covered writing and running specs in individual suites. Of course, the `ginkgo` CLI also supports running multiple suites with a single invocation on the command line. We'll close out this chapter on running specs by covering how Ginkgo runs multiple suites. @@ -5265,7 +5289,7 @@ The columns are: You can set a different output format with the `-format` flag. Accepted formats are `csv`, `indent`, and `json`. The `ident` format is like `csv`, but uses indentation to show the nesting of containers and specs. Both the `csv` and `json` formats can be read by another program, e.g., an editor plugin that displays a tree view of Ginkgo tests in a file, or presents a menu for the user to quickly navigate to a container or spec. -`ginkgo outline` is intended for integration with third-party libraries and applications. If you simply want to know how a suite will run without running it try `ginkgo -v --dry-run` instead. +`ginkgo outline` is intended for integration with third-party libraries and applications - however it has an important limitation. Since parses the go syntax tree it cannot identify specs that are dynamically generated. Nor does it capture run-time concerns such as which specs will be skipped by a given set of filters or the order in which specs will run. If you want a quick overview of such things you can use `ginkgo -v --dry-run` instead. If you want finer-grained control over the suite preview, you should use [`PreviewSpecs`](#previewing-specs). ### Other Subcommands diff --git a/dsl/core/core_dsl.go b/dsl/core/core_dsl.go index 06c8637a2..0796cee68 100644 --- a/dsl/core/core_dsl.go +++ b/dsl/core/core_dsl.go @@ -34,6 +34,7 @@ var GinkgoLabelFilter = ginkgo.GinkgoLabelFilter var PauseOutputInterception = ginkgo.PauseOutputInterception var ResumeOutputInterception = ginkgo.ResumeOutputInterception var RunSpecs = ginkgo.RunSpecs +var PreviewSpecs = ginkgo.PreviewSpecs var Skip = ginkgo.Skip var Fail = ginkgo.Fail var AbortSuite = ginkgo.AbortSuite diff --git a/integration/_fixtures/preview_fixture/preview_fixture_suite_test.go b/integration/_fixtures/preview_fixture/preview_fixture_suite_test.go new file mode 100644 index 000000000..5d50491ea --- /dev/null +++ b/integration/_fixtures/preview_fixture/preview_fixture_suite_test.go @@ -0,0 +1,41 @@ +package preview_fixture_test + +import ( + "fmt" + "os" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestPreviewFixture(t *testing.T) { + RegisterFailHandler(Fail) + if os.Getenv("RUN") == "true" { + RunSpecs(t, "PreviewFixture Suite", Label("suite-label")) + } + if os.Getenv("PREVIEW") == "true" { + report := PreviewSpecs("PreviewFixture Suite", Label("suite-label")) + for _, spec := range report.SpecReports { + fmt.Println(spec.State, spec.FullText()) + } + } +} + +var _ = Describe("specs", func() { + It("A", Label("elephant"), func() { + + }) + + It("B", Label("elephant"), func() { + + }) + + It("C", func() { + + }) + + It("D", func() { + + }) +}) diff --git a/integration/preview_test.go b/integration/preview_test.go new file mode 100644 index 000000000..b4d54a7d4 --- /dev/null +++ b/integration/preview_test.go @@ -0,0 +1,45 @@ +package integration_test + +import ( + "os" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/onsi/gomega/gbytes" + "github.com/onsi/gomega/gexec" +) + +var _ = Describe("Preview", func() { + BeforeEach(func() { + fm.MountFixture("preview") + }) + + It("previews the specs, honoring the passed in flags", func() { + os.Setenv("PREVIEW", "true") + DeferCleanup(os.Unsetenv, "PREVIEW") + session := startGinkgo(fm.PathTo("preview"), "--label-filter=elephant") + Eventually(session).Should(gexec.Exit(0)) + Ω(session).Should(gbytes.Say("passed specs A")) + Ω(session).Should(gbytes.Say("passed specs B")) + Ω(session).Should(gbytes.Say("skipped specs C")) + Ω(session).Should(gbytes.Say("skipped specs D")) + }) + + It("fails if running in parallel", func() { + os.Setenv("PREVIEW", "true") + DeferCleanup(os.Unsetenv, "PREVIEW") + session := startGinkgo(fm.PathTo("preview"), "--procs=2") + Eventually(session).Should(gexec.Exit(1)) + Ω(session.Err).Should(gbytes.Say(`Ginkgo only supports PreviewSpecs\(\) in serial mode\.`)) + }) + + It("fails if you attempt to both run and preview specs", func() { + os.Setenv("PREVIEW", "true") + DeferCleanup(os.Unsetenv, "PREVIEW") + os.Setenv("RUN", "true") + DeferCleanup(os.Unsetenv, "RUN") + session := startGinkgo(fm.PathTo("preview")) + Eventually(session).Should(gexec.Exit(1)) + Ω(session).Should(gbytes.Say(`It looks like you are calling RunSpecs and PreviewSpecs in the same invocation`)) + }) +}) diff --git a/internal/suite.go b/internal/suite.go index ea0d259d9..9c6847710 100644 --- a/internal/suite.go +++ b/internal/suite.go @@ -328,6 +328,16 @@ func (suite *Suite) CurrentSpecReport() types.SpecReport { return report } +// Only valid in the preview context. In general suite.report only includes +// the specs run by _this_ node - it is only at the end of the suite that +// the parallel reports are aggregated. However in the preview context we run +// in series and +func (suite *Suite) GetPreviewReport() types.Report { + suite.selectiveLock.Lock() + defer suite.selectiveLock.Unlock() + return suite.report +} + func (suite *Suite) AddReportEntry(entry ReportEntry) error { if suite.phase != PhaseRun { return types.GinkgoErrors.AddReportEntryNotDuringRunPhase(entry.Location) diff --git a/types/errors.go b/types/errors.go index 4fbdc3e9b..1be2259fe 100644 --- a/types/errors.go +++ b/types/errors.go @@ -70,6 +70,14 @@ func (g ginkgoErrors) RerunningSuite() error { } } +func (g ginkgoErrors) RunAndPreviewSuite() error { + return GinkgoError{ + Heading: "Running and Previewing Suite", + Message: formatter.F(`It looks like you are calling RunSpecs and PreviewSpecs in the same invocation of Ginkgo. Ginkgo does not currently support that. Please change your code to only call one or the other.`), + DocLink: "previewing-specs", + } +} + /* Tree construction errors */ func (g ginkgoErrors) PushingNodeInRunPhase(nodeType NodeType, cl CodeLocation) error { @@ -578,6 +586,13 @@ func (g ginkgoErrors) DryRunInParallelConfiguration() error { } } +func (g ginkgoErrors) PreviewInParallelConfiguration() error { + return GinkgoError{ + Heading: "Ginkgo only supports PreviewSpecs() in serial mode.", + Message: "Please try running ginkgo again, but without -p or -procs to ensure the suite is running in series.", + } +} + func (g ginkgoErrors) GracePeriodCannotBeZero() error { return GinkgoError{ Heading: "Ginkgo requires a positive --grace-period.",