Source file src/pkg/testing/testing.go
1 // Copyright 2009 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 // Package testing provides support for automated testing of Go packages. 6 // It is intended to be used in concert with the ``go test'' command, which automates 7 // execution of any function of the form 8 // func TestXxx(*testing.T) 9 // where Xxx can be any alphanumeric string (but the first letter must not be in 10 // [a-z]) and serves to identify the test routine. 11 // These TestXxx routines should be declared within the package they are testing. 12 // 13 // Functions of the form 14 // func BenchmarkXxx(*testing.B) 15 // are considered benchmarks, and are executed by the "go test" command when 16 // the -test.bench flag is provided. 17 // 18 // A sample benchmark function looks like this: 19 // func BenchmarkHello(b *testing.B) { 20 // for i := 0; i < b.N; i++ { 21 // fmt.Sprintf("hello") 22 // } 23 // } 24 // 25 // The benchmark package will vary b.N until the benchmark function lasts 26 // long enough to be timed reliably. The output 27 // testing.BenchmarkHello 10000000 282 ns/op 28 // means that the loop ran 10000000 times at a speed of 282 ns per loop. 29 // 30 // If a benchmark needs some expensive setup before running, the timer 31 // may be stopped: 32 // func BenchmarkBigLen(b *testing.B) { 33 // b.StopTimer() 34 // big := NewBig() 35 // b.StartTimer() 36 // for i := 0; i < b.N; i++ { 37 // big.Len() 38 // } 39 // } 40 // 41 // The package also runs and verifies example code. Example functions may 42 // include a concluding comment that begins with "Output:" and is compared with 43 // the standard output of the function when the tests are run, as in these 44 // examples of an example: 45 // 46 // func ExampleHello() { 47 // fmt.Println("hello") 48 // // Output: hello 49 // } 50 // 51 // func ExampleSalutations() { 52 // fmt.Println("hello, and") 53 // fmt.Println("goodbye") 54 // // Output: 55 // // hello, and 56 // // goodbye 57 // } 58 // 59 // Example functions without output comments are compiled but not executed. 60 // 61 // The naming convention to declare examples for a function F, a type T and 62 // method M on type T are: 63 // 64 // func ExampleF() { ... } 65 // func ExampleT() { ... } 66 // func ExampleT_M() { ... } 67 // 68 // Multiple example functions for a type/function/method may be provided by 69 // appending a distinct suffix to the name. The suffix must start with a 70 // lower-case letter. 71 // 72 // func ExampleF_suffix() { ... } 73 // func ExampleT_suffix() { ... } 74 // func ExampleT_M_suffix() { ... } 75 // 76 // The entire test file is presented as the example when it contains a single 77 // example function, at least one other function, type, variable, or constant 78 // declaration, and no test or benchmark functions. 79 package testing 80 81 import ( 82 "flag" 83 "fmt" 84 "os" 85 "runtime" 86 "runtime/pprof" 87 "strconv" 88 "strings" 89 "time" 90 ) 91 92 var ( 93 // The short flag requests that tests run more quickly, but its functionality 94 // is provided by test writers themselves. The testing package is just its 95 // home. The all.bash installation script sets it to make installation more 96 // efficient, but by default the flag is off so a plain "go test" will do a 97 // full test of the package. 98 short = flag.Bool("test.short", false, "run smaller test suite to save time") 99 100 // Report as tests are run; default is silent for success. 101 chatty = flag.Bool("test.v", false, "verbose: print additional output") 102 match = flag.String("test.run", "", "regular expression to select tests and examples to run") 103 memProfile = flag.String("test.memprofile", "", "write a memory profile to the named file after execution") 104 memProfileRate = flag.Int("test.memprofilerate", 0, "if >=0, sets runtime.MemProfileRate") 105 cpuProfile = flag.String("test.cpuprofile", "", "write a cpu profile to the named file during execution") 106 timeout = flag.Duration("test.timeout", 0, "if positive, sets an aggregate time limit for all tests") 107 cpuListStr = flag.String("test.cpu", "", "comma-separated list of number of CPUs to use for each test") 108 parallel = flag.Int("test.parallel", runtime.GOMAXPROCS(0), "maximum test parallelism") 109 110 haveExamples bool // are there examples? 111 112 cpuList []int 113 ) 114 115 // common holds the elements common between T and B and 116 // captures common methods such as Errorf. 117 type common struct { 118 output []byte // Output generated by test or benchmark. 119 failed bool // Test or benchmark has failed. 120 start time.Time // Time test or benchmark started 121 duration time.Duration 122 self interface{} // To be sent on signal channel when done. 123 signal chan interface{} // Output for serial tests. 124 } 125 126 // Short reports whether the -test.short flag is set. 127 func Short() bool { 128 return *short 129 } 130 131 // decorate inserts the final newline if needed and indentation tabs for formatting. 132 // If addFileLine is true, it also prefixes the string with the file and line of the call site. 133 func decorate(s string, addFileLine bool) string { 134 if addFileLine { 135 _, file, line, ok := runtime.Caller(3) // decorate + log + public function. 136 if ok { 137 // Truncate file name at last file name separator. 138 if index := strings.LastIndex(file, "/"); index >= 0 { 139 file = file[index+1:] 140 } else if index = strings.LastIndex(file, "\\"); index >= 0 { 141 file = file[index+1:] 142 } 143 } else { 144 file = "???" 145 line = 1 146 } 147 s = fmt.Sprintf("%s:%d: %s", file, line, s) 148 } 149 s = "\t" + s // Every line is indented at least one tab. 150 n := len(s) 151 if n > 0 && s[n-1] != '\n' { 152 s += "\n" 153 n++ 154 } 155 for i := 0; i < n-1; i++ { // -1 to avoid final newline 156 if s[i] == '\n' { 157 // Second and subsequent lines are indented an extra tab. 158 return s[0:i+1] + "\t" + decorate(s[i+1:n], false) 159 } 160 } 161 return s 162 } 163 164 // T is a type passed to Test functions to manage test state and support formatted test logs. 165 // Logs are accumulated during execution and dumped to standard error when done. 166 type T struct { 167 common 168 name string // Name of test. 169 startParallel chan bool // Parallel tests will wait on this. 170 } 171 172 // Fail marks the function as having failed but continues execution. 173 func (c *common) Fail() { c.failed = true } 174 175 // Failed returns whether the function has failed. 176 func (c *common) Failed() bool { return c.failed } 177 178 // FailNow marks the function as having failed and stops its execution. 179 // Execution will continue at the next test or benchmark. 180 func (c *common) FailNow() { 181 c.Fail() 182 183 // Calling runtime.Goexit will exit the goroutine, which 184 // will run the deferred functions in this goroutine, 185 // which will eventually run the deferred lines in tRunner, 186 // which will signal to the test loop that this test is done. 187 // 188 // A previous version of this code said: 189 // 190 // c.duration = ... 191 // c.signal <- c.self 192 // runtime.Goexit() 193 // 194 // This previous version duplicated code (those lines are in 195 // tRunner no matter what), but worse the goroutine teardown 196 // implicit in runtime.Goexit was not guaranteed to complete 197 // before the test exited. If a test deferred an important cleanup 198 // function (like removing temporary files), there was no guarantee 199 // it would run on a test failure. Because we send on c.signal during 200 // a top-of-stack deferred function now, we know that the send 201 // only happens after any other stacked defers have completed. 202 runtime.Goexit() 203 } 204 205 // log generates the output. It's always at the same stack depth. 206 func (c *common) log(s string) { 207 c.output = append(c.output, decorate(s, true)...) 208 } 209 210 // Log formats its arguments using default formatting, analogous to Println(), 211 // and records the text in the error log. 212 func (c *common) Log(args ...interface{}) { c.log(fmt.Sprintln(args...)) } 213 214 // Logf formats its arguments according to the format, analogous to Printf(), 215 // and records the text in the error log. 216 func (c *common) Logf(format string, args ...interface{}) { c.log(fmt.Sprintf(format, args...)) } 217 218 // Error is equivalent to Log() followed by Fail(). 219 func (c *common) Error(args ...interface{}) { 220 c.log(fmt.Sprintln(args...)) 221 c.Fail() 222 } 223 224 // Errorf is equivalent to Logf() followed by Fail(). 225 func (c *common) Errorf(format string, args ...interface{}) { 226 c.log(fmt.Sprintf(format, args...)) 227 c.Fail() 228 } 229 230 // Fatal is equivalent to Log() followed by FailNow(). 231 func (c *common) Fatal(args ...interface{}) { 232 c.log(fmt.Sprintln(args...)) 233 c.FailNow() 234 } 235 236 // Fatalf is equivalent to Logf() followed by FailNow(). 237 func (c *common) Fatalf(format string, args ...interface{}) { 238 c.log(fmt.Sprintf(format, args...)) 239 c.FailNow() 240 } 241 242 // Parallel signals that this test is to be run in parallel with (and only with) 243 // other parallel tests in this CPU group. 244 func (t *T) Parallel() { 245 t.signal <- (*T)(nil) // Release main testing loop 246 <-t.startParallel // Wait for serial tests to finish 247 } 248 249 // An internal type but exported because it is cross-package; part of the implementation 250 // of the "go test" command. 251 type InternalTest struct { 252 Name string 253 F func(*T) 254 } 255 256 func tRunner(t *T, test *InternalTest) { 257 t.start = time.Now() 258 259 // When this goroutine is done, either because test.F(t) 260 // returned normally or because a test failure triggered 261 // a call to runtime.Goexit, record the duration and send 262 // a signal saying that the test is done. 263 defer func() { 264 t.duration = time.Now().Sub(t.start) 265 // If the test panicked, print any test output before dying. 266 if err := recover(); err != nil { 267 t.report() 268 panic(err) 269 } 270 t.signal <- t 271 }() 272 273 test.F(t) 274 } 275 276 // An internal function but exported because it is cross-package; part of the implementation 277 // of the "go test" command. 278 func Main(matchString func(pat, str string) (bool, error), tests []InternalTest, benchmarks []InternalBenchmark, examples []InternalExample) { 279 flag.Parse() 280 parseCpuList() 281 282 before() 283 startAlarm() 284 haveExamples = len(examples) > 0 285 testOk := RunTests(matchString, tests) 286 exampleOk := RunExamples(matchString, examples) 287 if !testOk || !exampleOk { 288 fmt.Println("FAIL") 289 os.Exit(1) 290 } 291 fmt.Println("PASS") 292 stopAlarm() 293 RunBenchmarks(matchString, benchmarks) 294 after() 295 } 296 297 func (t *T) report() { 298 tstr := fmt.Sprintf("(%.2f seconds)", t.duration.Seconds()) 299 format := "--- %s: %s %s\n%s" 300 if t.failed { 301 fmt.Printf(format, "FAIL", t.name, tstr, t.output) 302 } else if *chatty { 303 fmt.Printf(format, "PASS", t.name, tstr, t.output) 304 } 305 } 306 307 func RunTests(matchString func(pat, str string) (bool, error), tests []InternalTest) (ok bool) { 308 ok = true 309 if len(tests) == 0 && !haveExamples { 310 fmt.Fprintln(os.Stderr, "testing: warning: no tests to run") 311 return 312 } 313 for _, procs := range cpuList { 314 runtime.GOMAXPROCS(procs) 315 // We build a new channel tree for each run of the loop. 316 // collector merges in one channel all the upstream signals from parallel tests. 317 // If all tests pump to the same channel, a bug can occur where a test 318 // kicks off a goroutine that Fails, yet the test still delivers a completion signal, 319 // which skews the counting. 320 var collector = make(chan interface{}) 321 322 numParallel := 0 323 startParallel := make(chan bool) 324 325 for i := 0; i < len(tests); i++ { 326 matched, err := matchString(*match, tests[i].Name) 327 if err != nil { 328 fmt.Fprintf(os.Stderr, "testing: invalid regexp for -test.run: %s\n", err) 329 os.Exit(1) 330 } 331 if !matched { 332 continue 333 } 334 testName := tests[i].Name 335 if procs != 1 { 336 testName = fmt.Sprintf("%s-%d", tests[i].Name, procs) 337 } 338 t := &T{ 339 common: common{ 340 signal: make(chan interface{}), 341 }, 342 name: testName, 343 startParallel: startParallel, 344 } 345 t.self = t 346 if *chatty { 347 fmt.Printf("=== RUN %s\n", t.name) 348 } 349 go tRunner(t, &tests[i]) 350 out := (<-t.signal).(*T) 351 if out == nil { // Parallel run. 352 go func() { 353 collector <- <-t.signal 354 }() 355 numParallel++ 356 continue 357 } 358 t.report() 359 ok = ok && !out.failed 360 } 361 362 running := 0 363 for numParallel+running > 0 { 364 if running < *parallel && numParallel > 0 { 365 startParallel <- true 366 running++ 367 numParallel-- 368 continue 369 } 370 t := (<-collector).(*T) 371 t.report() 372 ok = ok && !t.failed 373 running-- 374 } 375 } 376 return 377 } 378 379 // before runs before all testing. 380 func before() { 381 if *memProfileRate > 0 { 382 runtime.MemProfileRate = *memProfileRate 383 } 384 if *cpuProfile != "" { 385 f, err := os.Create(*cpuProfile) 386 if err != nil { 387 fmt.Fprintf(os.Stderr, "testing: %s", err) 388 return 389 } 390 if err := pprof.StartCPUProfile(f); err != nil { 391 fmt.Fprintf(os.Stderr, "testing: can't start cpu profile: %s", err) 392 f.Close() 393 return 394 } 395 // Could save f so after can call f.Close; not worth the effort. 396 } 397 398 } 399 400 // after runs after all testing. 401 func after() { 402 if *cpuProfile != "" { 403 pprof.StopCPUProfile() // flushes profile to disk 404 } 405 if *memProfile != "" { 406 f, err := os.Create(*memProfile) 407 if err != nil { 408 fmt.Fprintf(os.Stderr, "testing: %s", err) 409 return 410 } 411 if err = pprof.WriteHeapProfile(f); err != nil { 412 fmt.Fprintf(os.Stderr, "testing: can't write %s: %s", *memProfile, err) 413 } 414 f.Close() 415 } 416 } 417 418 var timer *time.Timer 419 420 // startAlarm starts an alarm if requested. 421 func startAlarm() { 422 if *timeout > 0 { 423 timer = time.AfterFunc(*timeout, alarm) 424 } 425 } 426 427 // stopAlarm turns off the alarm. 428 func stopAlarm() { 429 if *timeout > 0 { 430 timer.Stop() 431 } 432 } 433 434 // alarm is called if the timeout expires. 435 func alarm() { 436 panic("test timed out") 437 } 438 439 func parseCpuList() { 440 if len(*cpuListStr) == 0 { 441 cpuList = append(cpuList, runtime.GOMAXPROCS(-1)) 442 } else { 443 for _, val := range strings.Split(*cpuListStr, ",") { 444 cpu, err := strconv.Atoi(val) 445 if err != nil || cpu <= 0 { 446 fmt.Fprintf(os.Stderr, "testing: invalid value %q for -test.cpu", val) 447 os.Exit(1) 448 } 449 cpuList = append(cpuList, cpu) 450 } 451 } 452 }