1
2
3
4
5
6
7
8
9
10
11
12
13 package com.eviware.soapui.tools;
14
15 import java.io.File;
16 import java.io.FileOutputStream;
17 import java.io.PrintWriter;
18 import java.io.StringWriter;
19 import java.util.ArrayList;
20 import java.util.Arrays;
21 import java.util.HashMap;
22 import java.util.List;
23 import java.util.Map;
24
25 import org.apache.commons.cli.CommandLine;
26
27 import com.eviware.soapui.SoapUI;
28 import com.eviware.soapui.impl.wsdl.WsdlProject;
29 import com.eviware.soapui.impl.wsdl.WsdlTestSuite;
30 import com.eviware.soapui.impl.wsdl.testcase.WsdlProjectRunner;
31 import com.eviware.soapui.impl.wsdl.testcase.WsdlTestCase;
32 import com.eviware.soapui.impl.wsdl.testcase.WsdlTestCaseRunner;
33 import com.eviware.soapui.impl.wsdl.testcase.WsdlTestSuiteRunner;
34 import com.eviware.soapui.impl.wsdl.teststeps.WsdlRunTestCaseTestStep;
35 import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestStep;
36 import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestStepResult;
37 import com.eviware.soapui.model.iface.Attachment;
38 import com.eviware.soapui.model.iface.MessageExchange;
39 import com.eviware.soapui.model.project.ProjectFactoryRegistry;
40 import com.eviware.soapui.model.support.ModelSupport;
41 import com.eviware.soapui.model.support.ProjectRunListenerAdapter;
42 import com.eviware.soapui.model.testsuite.Assertable;
43 import com.eviware.soapui.model.testsuite.AssertionError;
44 import com.eviware.soapui.model.testsuite.ProjectRunContext;
45 import com.eviware.soapui.model.testsuite.ProjectRunner;
46 import com.eviware.soapui.model.testsuite.TestAssertion;
47 import com.eviware.soapui.model.testsuite.TestCase;
48 import com.eviware.soapui.model.testsuite.TestCaseRunContext;
49 import com.eviware.soapui.model.testsuite.TestCaseRunner;
50 import com.eviware.soapui.model.testsuite.TestStep;
51 import com.eviware.soapui.model.testsuite.TestStepResult;
52 import com.eviware.soapui.model.testsuite.TestSuite;
53 import com.eviware.soapui.model.testsuite.TestSuiteRunner;
54 import com.eviware.soapui.model.testsuite.Assertable.AssertionStatus;
55 import com.eviware.soapui.model.testsuite.TestRunner.Status;
56 import com.eviware.soapui.model.testsuite.TestStepResult.TestStepStatus;
57 import com.eviware.soapui.report.JUnitReportCollector;
58 import com.eviware.soapui.support.StringUtils;
59 import com.eviware.soapui.support.Tools;
60 import com.eviware.soapui.support.types.StringToObjectMap;
61
62 /***
63 * Standalone test-runner used from maven-plugin, can also be used from
64 * command-line (see xdocs) or directly from other classes.
65 * <p>
66 * For standalone usage, set the project file (with setProjectFile) and other
67 * desired properties before calling run
68 * </p>
69 *
70 * @author Ole.Matzura
71 */
72
73 public class SoapUITestCaseRunner extends AbstractSoapUITestRunner
74 {
75 public static final String SOAPUI_EXPORT_SEPARATOR = "soapui.export.separator";
76
77 public static final String TITLE = "soapUI " + SoapUI.SOAPUI_VERSION + " TestCase Runner";
78
79 private String testSuite;
80 private String testCase;
81 private List<TestAssertion> assertions = new ArrayList<TestAssertion>();
82 private Map<TestAssertion, WsdlTestStepResult> assertionResults = new HashMap<TestAssertion, WsdlTestStepResult>();
83
84
85 private List<TestCase> failedTests = new ArrayList<TestCase>();
86
87 private int testSuiteCount;
88 private int testCaseCount;
89 private int testStepCount;
90 private int testAssertionCount;
91
92 private boolean printReport;
93 private boolean exportAll;
94 private boolean ignoreErrors;
95 private boolean junitReport;
96 private int exportCount;
97 private JUnitReportCollector reportCollector;
98
99 private String projectPassword;
100 private boolean saveAfterRun;
101
102 /***
103 * Runs the tests in the specified soapUI project file, see soapUI xdocs for
104 * details.
105 *
106 * @param args
107 * @throws Exception
108 */
109
110 public static void main( String[] args ) throws Exception
111 {
112 System.exit( new SoapUITestCaseRunner().runFromCommandLine( args ) );
113 }
114
115 protected boolean processCommandLine( CommandLine cmd )
116 {
117 if( cmd.hasOption( "e" ) )
118 setEndpoint( cmd.getOptionValue( "e" ) );
119
120 if( cmd.hasOption( "s" ) )
121 setTestSuite( getCommandLineOptionSubstSpace( cmd, "s" ) );
122
123 if( cmd.hasOption( "c" ) )
124 setTestCase( getCommandLineOptionSubstSpace( cmd, "c" ) );
125
126 if( cmd.hasOption( "u" ) )
127 setUsername( cmd.getOptionValue( "u" ) );
128
129 if( cmd.hasOption( "p" ) )
130 setPassword( cmd.getOptionValue( "p" ) );
131
132 if( cmd.hasOption( "w" ) )
133 setWssPasswordType( cmd.getOptionValue( "w" ) );
134
135 if( cmd.hasOption( "d" ) )
136 setDomain( cmd.getOptionValue( "d" ) );
137
138 if( cmd.hasOption( "h" ) )
139 setHost( cmd.getOptionValue( "h" ) );
140
141 if( cmd.hasOption( "f" ) )
142 setOutputFolder( getCommandLineOptionSubstSpace( cmd, "f" ) );
143
144 if( cmd.hasOption( "t" ) )
145 setSettingsFile( getCommandLineOptionSubstSpace( cmd, "t" ) );
146
147 if( cmd.hasOption( "x" ) )
148 {
149 setProjectPassword( cmd.getOptionValue( "x" ) );
150 }
151
152 if( cmd.hasOption( "v" ) )
153 {
154 setSoapUISettingsPassword( cmd.getOptionValue( "v" ) );
155 }
156
157 if( cmd.hasOption( "D" ) )
158 {
159 setSystemProperties( cmd.getOptionValues( "D" ) );
160 }
161
162 if( cmd.hasOption( "G" ) )
163 {
164 setGlobalProperties( cmd.getOptionValues( "G" ) );
165 }
166
167 if( cmd.hasOption( "P" ) )
168 {
169 setProjectProperties( cmd.getOptionValues( "P" ) );
170 }
171
172 setIgnoreError( cmd.hasOption( "I" ) );
173 setEnableUI( cmd.hasOption( "i" ) );
174 setPrintReport( cmd.hasOption( "r" ) );
175 setExportAll( cmd.hasOption( "a" ) );
176 if( cmd.hasOption( "A" ) )
177 {
178 setExportAll( true );
179 System.setProperty( SOAPUI_EXPORT_SEPARATOR, File.separator );
180 }
181
182 setJUnitReport( cmd.hasOption( "j" ) );
183 setSaveAfterRun( cmd.hasOption( "S" ) );
184
185 return true;
186 }
187
188 public void setSaveAfterRun( boolean saveAfterRun )
189 {
190 this.saveAfterRun = saveAfterRun;
191 }
192
193 public void setProjectPassword( String projectPassword )
194 {
195 this.projectPassword = projectPassword;
196 }
197
198 public String getProjectPassword()
199 {
200 return projectPassword;
201 }
202
203 protected SoapUIOptions initCommandLineOptions()
204 {
205 SoapUIOptions options = new SoapUIOptions( "testrunner" );
206 options.addOption( "e", true, "Sets the endpoint" );
207 options.addOption( "s", true, "Sets the testsuite" );
208 options.addOption( "c", true, "Sets the testcase" );
209 options.addOption( "u", true, "Sets the username" );
210 options.addOption( "p", true, "Sets the password" );
211 options.addOption( "w", true, "Sets the WSS password type, either 'Text' or 'Digest'" );
212 options.addOption( "i", false, "Enables Swing UI for scripts" );
213 options.addOption( "d", true, "Sets the domain" );
214 options.addOption( "h", true, "Sets the host" );
215 options.addOption( "r", false, "Prints a small summary report" );
216 options.addOption( "f", true, "Sets the output folder to export results to" );
217 options.addOption( "j", false, "Sets the output to include JUnit XML reports" );
218 options.addOption( "a", false, "Turns on exporting of all results" );
219 options.addOption( "A", false, "Turns on exporting of all results using folders instead of long filenames" );
220 options.addOption( "t", true, "Sets the soapui-settings.xml file to use" );
221 options.addOption( "x", true, "Sets project password for decryption if project is encrypted" );
222 options.addOption( "v", true, "Sets password for soapui-settings.xml file" );
223 options.addOption( "D", true, "Sets system property with name=value" );
224 options.addOption( "G", true, "Sets global property with name=value" );
225 options.addOption( "P", true, "Sets or overrides project property with name=value" );
226 options.addOption( "I", false, "Do not stop if error occurs, ignore them" );
227 options.addOption( "S", false, "Saves the project after running the tests" );
228
229 return options;
230 }
231
232 /***
233 * Add console appender to groovy log
234 */
235
236 public void setExportAll( boolean exportAll )
237 {
238 this.exportAll = exportAll;
239 }
240
241 public void setJUnitReport( boolean junitReport )
242 {
243 this.junitReport = junitReport;
244 if( junitReport )
245 reportCollector = new JUnitReportCollector();
246 }
247
248 public SoapUITestCaseRunner()
249 {
250 super( SoapUITestCaseRunner.TITLE );
251 }
252
253 public SoapUITestCaseRunner( String title )
254 {
255 super( title );
256 }
257
258 /***
259 * Controls if a short test summary should be printed after the test runs
260 *
261 * @param printReport
262 * a flag controlling if a summary should be printed
263 */
264
265 public void setPrintReport( boolean printReport )
266 {
267 this.printReport = printReport;
268 }
269
270 public void setIgnoreError( boolean ignoreErrors )
271 {
272 this.ignoreErrors = ignoreErrors;
273 }
274
275 public boolean runRunner() throws Exception
276 {
277 initGroovyLog();
278
279 assertions.clear();
280
281 String projectFile = getProjectFile();
282
283 WsdlProject project = ( WsdlProject )ProjectFactoryRegistry.getProjectFactory( "wsdl" ).createNew( projectFile,
284 getProjectPassword() );
285
286 if( project.isDisabled() )
287 throw new Exception( "Failed to load soapUI project file [" + projectFile + "]" );
288
289 initProject( project );
290 ensureOutputFolder( project );
291
292 log.info( "Running soapUI tests in project [" + project.getName() + "]" );
293
294 long startTime = System.nanoTime();
295
296 List<TestCase> testCasesToRun = new ArrayList<TestCase>();
297
298
299
300 for( int c = 0; c < project.getTestSuiteCount(); c++ )
301 {
302 TestSuite suite = project.getTestSuiteAt( c );
303 for( int i = 0; i < suite.getTestCaseCount(); i++ )
304 {
305 TestCase tc = suite.getTestCaseAt( i );
306 if( ( testSuite == null || suite.getName().equals( suite.getName() ) ) && testCase != null
307 && tc.getName().equals( testCase ) )
308 testCasesToRun.add( tc );
309
310 addListeners( tc );
311 }
312 }
313
314
315 if( testCasesToRun.size() > 0 )
316 {
317 for( TestCase testCase : testCasesToRun )
318 runTestCase( ( WsdlTestCase )testCase );
319 }
320 else if( testSuite != null )
321 {
322 runSuite( project.getTestSuiteByName( testSuite ) );
323 }
324 else
325 {
326 runProject( project );
327 }
328
329 long timeTaken = ( System.nanoTime() - startTime ) / 1000000;
330
331 if( printReport )
332 {
333 printReport( timeTaken );
334 }
335
336 exportReports( project );
337
338 if( saveAfterRun && !project.isRemote() )
339 {
340 try
341 {
342 project.save();
343 }
344 catch( Throwable t )
345 {
346 log.error( "Failed to save project", t );
347 }
348 }
349
350 if( ( assertions.size() > 0 || failedTests.size() > 0 ) && !ignoreErrors )
351 {
352 throwFailureException();
353 }
354
355 return true;
356 }
357
358 protected void runProject( WsdlProject project )
359 {
360
361 InternalProjectRunListener projectRunListener = new InternalProjectRunListener();
362 project.addProjectRunListener( projectRunListener );
363
364 try
365 {
366 log.info( ( "Running Project [" + project.getName() + "], runType = " + project.getRunType() ) );
367 WsdlProjectRunner runner = project.run( new StringToObjectMap(), false );
368 log.info( "Project [" + project.getName() + "] finished with status [" + runner.getStatus() + "] in "
369 + runner.getTimeTaken() + "ms" );
370 }
371 catch( Exception e )
372 {
373 e.printStackTrace();
374 }
375 finally
376 {
377 project.removeProjectRunListener( projectRunListener );
378 }
379 }
380
381 protected void initProject( WsdlProject project ) throws Exception
382 {
383 initProjectProperties( project );
384 }
385
386 protected void exportReports( WsdlProject project ) throws Exception
387 {
388 if( junitReport )
389 {
390 exportJUnitReports( reportCollector, getAbsoluteOutputFolder( project ), project );
391 }
392 }
393
394 protected void addListeners( TestCase tc )
395 {
396 tc.addTestRunListener( this );
397 if( junitReport )
398 tc.addTestRunListener( reportCollector );
399 }
400
401 protected void throwFailureException() throws Exception
402 {
403 StringBuffer buf = new StringBuffer();
404
405 for( int c = 0; c < assertions.size(); c++ )
406 {
407 TestAssertion assertion = assertions.get( c );
408 Assertable assertable = assertion.getAssertable();
409 if( assertable instanceof WsdlTestStep )
410 failedTests.remove( ( ( WsdlTestStep )assertable ).getTestCase() );
411
412 buf.append( assertion.getName() + " in [" + assertable.getModelItem().getName() + "] failed;\n" );
413 buf.append( Arrays.toString( assertion.getErrors() ) + "\n" );
414
415 WsdlTestStepResult result = assertionResults.get( assertion );
416 StringWriter stringWriter = new StringWriter();
417 PrintWriter writer = new PrintWriter( stringWriter );
418 result.writeTo( writer );
419 buf.append( stringWriter.toString() );
420 }
421
422 while( !failedTests.isEmpty() )
423 {
424 buf.append( "TestCase [" + failedTests.remove( 0 ).getName() + "] failed without assertions\n" );
425 }
426
427 throw new Exception( buf.toString() );
428 }
429
430 public void exportJUnitReports( JUnitReportCollector collector, String folder, WsdlProject project )
431 throws Exception
432 {
433 collector.saveReports( folder == null ? "" : folder );
434 }
435
436 public void printReport( long timeTaken )
437 {
438 System.out.println();
439 System.out.println( "SoapUI " + SoapUI.SOAPUI_VERSION + " TestCaseRunner Summary" );
440 System.out.println( "-----------------------------" );
441 System.out.println( "Time Taken: " + timeTaken + "ms" );
442 System.out.println( "Total TestSuites: " + testSuiteCount );
443 System.out.println( "Total TestCases: " + testCaseCount + " (" + failedTests.size() + " failed)" );
444 System.out.println( "Total TestSteps: " + testStepCount );
445 System.out.println( "Total Request Assertions: " + testAssertionCount );
446 System.out.println( "Total Failed Assertions: " + assertions.size() );
447 System.out.println( "Total Exported Results: " + exportCount );
448 }
449
450 /***
451 * Run tests in the specified TestSuite
452 *
453 * @param suite
454 * the TestSuite to run
455 */
456
457 protected void runSuite( WsdlTestSuite suite )
458 {
459 try
460 {
461 log.info( ( "Running TestSuite [" + suite.getName() + "], runType = " + suite.getRunType() ) );
462 WsdlTestSuiteRunner runner = suite.run( new StringToObjectMap(), false );
463 log.info( "TestSuite [" + suite.getName() + "] finished with status [" + runner.getStatus() + "] in "
464 + ( runner.getTimeTaken() ) + "ms" );
465 }
466 catch( Exception e )
467 {
468 e.printStackTrace();
469 }
470 finally
471 {
472 testSuiteCount++ ;
473 }
474 }
475
476 /***
477 * Runs the specified TestCase
478 *
479 * @param testCase
480 * the testcase to run
481 * @param context
482 */
483
484 protected void runTestCase( WsdlTestCase testCase )
485 {
486 try
487 {
488 log.info( "Running TestCase [" + testCase.getName() + "]" );
489 WsdlTestCaseRunner runner = testCase.run( new StringToObjectMap(), false );
490 log.info( "TestCase [" + testCase.getName() + "] finished with status [" + runner.getStatus() + "] in "
491 + ( runner.getTimeTaken() ) + "ms" );
492 }
493 catch( Exception e )
494 {
495 e.printStackTrace();
496 }
497 }
498
499 /***
500 * Sets the testcase to run
501 *
502 * @param testCase
503 * the testcase to run
504 */
505
506 public void setTestCase( String testCase )
507 {
508 this.testCase = testCase;
509 }
510
511 /***
512 * Sets the TestSuite to run. If not set all TestSuites in the specified
513 * project file are run
514 *
515 * @param testSuite
516 * the testSuite to run.
517 */
518
519 public void setTestSuite( String testSuite )
520 {
521 this.testSuite = testSuite;
522 }
523
524 public void beforeRun( TestCaseRunner testRunner, TestCaseRunContext runContext )
525 {
526 log.info( "Running soapUI testcase [" + testRunner.getTestCase().getName() + "]" );
527 }
528
529 public void beforeStep( TestCaseRunner testRunner, TestCaseRunContext runContext, TestStep currentStep )
530 {
531 super.beforeStep( testRunner, runContext, currentStep );
532
533 if( currentStep != null )
534 log.info( "running step [" + currentStep.getName() + "]" );
535 }
536
537 public void afterStep( TestCaseRunner testRunner, TestCaseRunContext runContext, TestStepResult result )
538 {
539 super.afterStep( testRunner, runContext, result );
540 TestStep currentStep = runContext.getCurrentStep();
541
542 if( currentStep instanceof Assertable )
543 {
544 Assertable requestStep = ( Assertable )currentStep;
545 for( int c = 0; c < requestStep.getAssertionCount(); c++ )
546 {
547 TestAssertion assertion = requestStep.getAssertionAt( c );
548 log.info( "Assertion [" + assertion.getName() + "] has status " + assertion.getStatus() );
549 if( assertion.getStatus() == AssertionStatus.FAILED )
550 {
551 for( AssertionError error : assertion.getErrors() )
552 log.error( "ASSERTION FAILED -> " + error.getMessage() );
553
554 assertions.add( assertion );
555 assertionResults.put( assertion, ( WsdlTestStepResult )result );
556 }
557
558 testAssertionCount++ ;
559 }
560 }
561
562 String countPropertyName = currentStep.getName() + " run count";
563 Long count = ( Long )runContext.getProperty( countPropertyName );
564 if( count == null )
565 {
566 count = new Long( 0 );
567 }
568
569 runContext.setProperty( countPropertyName, new Long( count.longValue() + 1 ) );
570
571 if( result.getStatus() == TestStepStatus.FAILED || exportAll )
572 {
573 try
574 {
575 String exportSeparator = System.getProperty( SOAPUI_EXPORT_SEPARATOR, "-" );
576
577 TestCase tc = currentStep.getTestCase();
578 String nameBase = StringUtils.createFileName( tc.getTestSuite().getName(), '_' ) + exportSeparator
579 + StringUtils.createFileName( tc.getName(), '_' ) + exportSeparator
580 + StringUtils.createFileName( currentStep.getName(), '_' ) + "-" + count.longValue() + "-"
581 + result.getStatus();
582
583 WsdlTestCaseRunner callingTestCaseRunner = ( WsdlTestCaseRunner )runContext
584 .getProperty( "#CallingTestCaseRunner#" );
585
586 if( callingTestCaseRunner != null )
587 {
588 WsdlTestCase ctc = callingTestCaseRunner.getTestCase();
589 WsdlRunTestCaseTestStep runTestCaseTestStep = ( WsdlRunTestCaseTestStep )runContext
590 .getProperty( "#CallingRunTestCaseStep#" );
591
592 nameBase = StringUtils.createFileName( ctc.getTestSuite().getName(), '_' ) + exportSeparator
593 + StringUtils.createFileName( ctc.getName(), '_' ) + exportSeparator
594 + StringUtils.createFileName( runTestCaseTestStep.getName(), '_' ) + exportSeparator
595 + StringUtils.createFileName( tc.getTestSuite().getName(), '_' ) + exportSeparator
596 + StringUtils.createFileName( tc.getName(), '_' ) + exportSeparator
597 + StringUtils.createFileName( currentStep.getName(), '_' ) + "-" + count.longValue() + "-"
598 + result.getStatus();
599 }
600
601 String absoluteOutputFolder = getAbsoluteOutputFolder( ModelSupport.getModelItemProject( tc ) );
602 String fileName = absoluteOutputFolder + File.separator + nameBase + ".txt";
603
604 if( result.getStatus() == TestStepStatus.FAILED )
605 log.error( currentStep.getName() + " failed, exporting to [" + fileName + "]" );
606
607 new File( fileName ).getParentFile().mkdirs();
608
609 PrintWriter writer = new PrintWriter( fileName );
610 result.writeTo( writer );
611 writer.close();
612
613
614 if( result instanceof MessageExchange )
615 {
616 Attachment[] attachments = ( ( MessageExchange )result ).getResponseAttachments();
617 if( attachments != null && attachments.length > 0 )
618 {
619 for( int c = 0; c < attachments.length; c++ )
620 {
621 fileName = nameBase + "-attachment-" + ( c + 1 ) + ".";
622
623 Attachment attachment = attachments[c];
624 String contentType = attachment.getContentType();
625 if( !"application/octet-stream".equals( contentType ) && contentType != null
626 && contentType.indexOf( '/' ) != -1 )
627 {
628 fileName += contentType.substring( contentType.lastIndexOf( '/' ) + 1 );
629 }
630 else
631 {
632 fileName += "dat";
633 }
634
635 fileName = absoluteOutputFolder + File.separator + fileName;
636
637 FileOutputStream outFile = new FileOutputStream( fileName );
638 Tools.writeAll( outFile, attachment.getInputStream() );
639 outFile.close();
640 }
641 }
642 }
643
644 exportCount++ ;
645 }
646 catch( Exception e )
647 {
648 log.error( "Error saving failed result: " + e, e );
649 }
650 }
651
652 testStepCount++ ;
653 }
654
655 public void afterRun( TestCaseRunner testRunner, TestCaseRunContext runContext )
656 {
657 log.info( "Finished running soapUI testcase [" + testRunner.getTestCase().getName() + "], time taken: "
658 + testRunner.getTimeTaken() + "ms, status: " + testRunner.getStatus() );
659
660 if( testRunner.getStatus() == Status.FAILED )
661 {
662 failedTests.add( testRunner.getTestCase() );
663 }
664
665 testCaseCount++ ;
666 }
667
668 private class InternalProjectRunListener extends ProjectRunListenerAdapter
669 {
670 public void afterTestSuite( ProjectRunner projectRunner, ProjectRunContext runContext, TestSuiteRunner testRunner )
671 {
672 testSuiteCount++ ;
673 }
674 }
675 }