1
2
3
4
5
6
7
8
9
10
11
12
13 package com.eviware.soapui.tools;
14
15 import java.io.File;
16 import java.io.FileOutputStream;
17 import java.io.PrintWriter;
18 import java.io.StringWriter;
19 import java.util.ArrayList;
20 import java.util.Arrays;
21 import java.util.HashMap;
22 import java.util.List;
23 import java.util.Map;
24
25 import org.apache.commons.cli.CommandLine;
26
27 import com.eviware.soapui.SoapUI;
28 import com.eviware.soapui.impl.wsdl.WsdlProject;
29 import com.eviware.soapui.impl.wsdl.WsdlTestSuite;
30 import com.eviware.soapui.impl.wsdl.testcase.WsdlProjectRunner;
31 import com.eviware.soapui.impl.wsdl.testcase.WsdlTestCase;
32 import com.eviware.soapui.impl.wsdl.testcase.WsdlTestCaseRunner;
33 import com.eviware.soapui.impl.wsdl.testcase.WsdlTestSuiteRunner;
34 import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestRequestStepResult;
35 import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestStep;
36 import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestStepResult;
37 import com.eviware.soapui.model.iface.Attachment;
38 import com.eviware.soapui.model.project.ProjectFactoryRegistry;
39 import com.eviware.soapui.model.support.ModelSupport;
40 import com.eviware.soapui.model.testsuite.Assertable;
41 import com.eviware.soapui.model.testsuite.AssertionError;
42 import com.eviware.soapui.model.testsuite.TestAssertion;
43 import com.eviware.soapui.model.testsuite.TestCase;
44 import com.eviware.soapui.model.testsuite.TestCaseRunContext;
45 import com.eviware.soapui.model.testsuite.TestCaseRunner;
46 import com.eviware.soapui.model.testsuite.TestStep;
47 import com.eviware.soapui.model.testsuite.TestStepResult;
48 import com.eviware.soapui.model.testsuite.TestSuite;
49 import com.eviware.soapui.model.testsuite.Assertable.AssertionStatus;
50 import com.eviware.soapui.model.testsuite.TestRunner.Status;
51 import com.eviware.soapui.model.testsuite.TestStepResult.TestStepStatus;
52 import com.eviware.soapui.report.JUnitReportCollector;
53 import com.eviware.soapui.support.StringUtils;
54 import com.eviware.soapui.support.Tools;
55 import com.eviware.soapui.support.types.StringToObjectMap;
56
57 /***
58 * Standalone test-runner used from maven-plugin, can also be used from
59 * command-line (see xdocs) or directly from other classes.
60 * <p>
61 * For standalone usage, set the project file (with setProjectFile) and other
62 * desired properties before calling run
63 * </p>
64 *
65 * @author Ole.Matzura
66 */
67
68 public class SoapUITestCaseRunner extends AbstractSoapUITestRunner
69 {
70 public static final String TITLE = "soapUI " + SoapUI.SOAPUI_VERSION + " TestCase Runner";
71
72 private String testSuite;
73 private String testCase;
74 private List<TestAssertion> assertions = new ArrayList<TestAssertion>();
75 private Map<TestAssertion, WsdlTestStepResult> assertionResults = new HashMap<TestAssertion, WsdlTestStepResult>();
76
77 private List<TestCase> failedTests = new ArrayList<TestCase>();
78
79 private int testSuiteCount;
80 private int testCaseCount;
81 private int testStepCount;
82 private int testAssertionCount;
83
84 private boolean printReport;
85 private boolean exportAll;
86 private boolean ignoreErrors;
87 private boolean junitReport;
88 private int exportCount;
89 private JUnitReportCollector reportCollector;
90
91 private String projectPassword;
92 private boolean saveAfterRun;
93
94 /***
95 * Runs the tests in the specified soapUI project file, see soapUI xdocs for
96 * details.
97 *
98 * @param args
99 * @throws Exception
100 */
101
102 public static void main( String[] args ) throws Exception
103 {
104 System.exit( new SoapUITestCaseRunner().runFromCommandLine( args ));
105 }
106
107 protected boolean processCommandLine( CommandLine cmd )
108 {
109 if( cmd.hasOption( "e" ) )
110 setEndpoint( cmd.getOptionValue( "e" ) );
111
112 if( cmd.hasOption( "s" ) )
113 setTestSuite( getCommandLineOptionSubstSpace( cmd, "s" ) );
114
115 if( cmd.hasOption( "c" ) )
116 setTestCase( getCommandLineOptionSubstSpace( cmd, "c" ) );
117
118 if( cmd.hasOption( "u" ) )
119 setUsername( cmd.getOptionValue( "u" ) );
120
121 if( cmd.hasOption( "p" ) )
122 setPassword( cmd.getOptionValue( "p" ) );
123
124 if( cmd.hasOption( "w" ) )
125 setWssPasswordType( cmd.getOptionValue( "w" ) );
126
127 if( cmd.hasOption( "d" ) )
128 setDomain( cmd.getOptionValue( "d" ) );
129
130 if( cmd.hasOption( "h" ) )
131 setHost( cmd.getOptionValue( "h" ) );
132
133 if( cmd.hasOption( "f" ) )
134 setOutputFolder( getCommandLineOptionSubstSpace( cmd, "f" ) );
135
136 if( cmd.hasOption( "t" ) )
137 setSettingsFile( getCommandLineOptionSubstSpace( cmd, "t" ) );
138
139 if( cmd.hasOption( "x" ) )
140 {
141 setProjectPassword( cmd.getOptionValue( "x" ) );
142 }
143
144 if( cmd.hasOption( "v" ) )
145 {
146 setSoapUISettingsPassword( cmd.getOptionValue( "v" ) );
147 }
148
149 if( cmd.hasOption( "D" ) )
150 {
151 setSystemProperties( cmd.getOptionValues( "D" ) );
152 }
153
154 if( cmd.hasOption( "G" ) )
155 {
156 setGlobalProperties( cmd.getOptionValues( "G" ) );
157 }
158
159 if( cmd.hasOption( "P" ) )
160 {
161 setProjectProperties( cmd.getOptionValues( "P" ) );
162 }
163
164 setIgnoreError( cmd.hasOption( "I" ) );
165 setEnableUI( cmd.hasOption( "i" ) );
166 setPrintReport( cmd.hasOption( "r" ) );
167 setExportAll( cmd.hasOption( "a" ) );
168 setJUnitReport( cmd.hasOption( "j" ) );
169 setSaveAfterRun( cmd.hasOption( "S" ) );
170
171 return true;
172 }
173
174 public void setSaveAfterRun( boolean saveAfterRun )
175 {
176 this.saveAfterRun = saveAfterRun;
177 }
178
179 public void setProjectPassword( String projectPassword )
180 {
181 this.projectPassword = projectPassword;
182 }
183
184 public String getProjectPassword()
185 {
186 return projectPassword;
187 }
188
189 protected SoapUIOptions initCommandLineOptions()
190 {
191 SoapUIOptions options = new SoapUIOptions( "testrunner" );
192 options.addOption( "e", true, "Sets the endpoint" );
193 options.addOption( "s", true, "Sets the testsuite" );
194 options.addOption( "c", true, "Sets the testcase" );
195 options.addOption( "u", true, "Sets the username" );
196 options.addOption( "p", true, "Sets the password" );
197 options.addOption( "w", true, "Sets the WSS password type, either 'Text' or 'Digest'" );
198 options.addOption( "i", false, "Enables Swing UI for scripts" );
199 options.addOption( "d", true, "Sets the domain" );
200 options.addOption( "h", true, "Sets the host" );
201 options.addOption( "r", false, "Prints a small summary report" );
202 options.addOption( "f", true, "Sets the output folder to export results to" );
203 options.addOption( "j", false, "Sets the output to include JUnit XML reports" );
204 options.addOption( "a", false, "Turns on exporting of all results" );
205 options.addOption( "t", true, "Sets the soapui-settings.xml file to use" );
206 options.addOption( "x", true, "Sets project password for decryption if project is encrypted" );
207 options.addOption( "v", true, "Sets password for soapui-settings.xml file" );
208 options.addOption( "D", true, "Sets system property with name=value" );
209 options.addOption( "G", true, "Sets global property with name=value" );
210 options.addOption( "P", true, "Sets or overrides project property with name=value" );
211 options.addOption( "I", false, "Do not stop if error occurs, ignore them" );
212 options.addOption( "S", false, "Saves the project after running the tests" );
213
214 return options;
215 }
216
217 /***
218 * Add console appender to groovy log
219 */
220
221 public void setExportAll( boolean exportAll )
222 {
223 this.exportAll = exportAll;
224 }
225
226 public void setJUnitReport( boolean junitReport )
227 {
228 this.junitReport = junitReport;
229 if( junitReport )
230 reportCollector = new JUnitReportCollector();
231 }
232
233 public SoapUITestCaseRunner()
234 {
235 super( SoapUITestCaseRunner.TITLE );
236 }
237
238 public SoapUITestCaseRunner( String title )
239 {
240 super( title );
241 }
242
243 /***
244 * Controls if a short test summary should be printed after the test runs
245 *
246 * @param printReport
247 * a flag controlling if a summary should be printed
248 */
249
250 public void setPrintReport( boolean printReport )
251 {
252 this.printReport = printReport;
253 }
254
255 public void setIgnoreError( boolean ignoreErrors )
256 {
257 this.ignoreErrors = ignoreErrors;
258 }
259
260 public boolean runRunner() throws Exception
261 {
262 initGroovyLog();
263
264 assertions.clear();
265
266 String projectFile = getProjectFile();
267
268 WsdlProject project = ( WsdlProject )ProjectFactoryRegistry.getProjectFactory( "wsdl" ).createNew( projectFile,
269 getProjectPassword() );
270
271 if( project.isDisabled() )
272 throw new Exception( "Failed to load soapUI project file [" + projectFile + "]" );
273
274 initProject( project );
275 ensureOutputFolder( project );
276
277 log.info( "Running soapUI tests in project [" + project.getName() + "]" );
278
279 long startTime = System.nanoTime();
280
281 List<TestCase> testCasesToRun = new ArrayList<TestCase>();
282
283
284
285 for( int c = 0; c < project.getTestSuiteCount(); c++ )
286 {
287 TestSuite suite = project.getTestSuiteAt( c );
288 for( int i = 0; i < suite.getTestCaseCount(); i++ )
289 {
290 TestCase tc = suite.getTestCaseAt( i );
291 if( testSuite == null && testCase != null && tc.getName().equals( testCase ))
292 testCasesToRun.add( tc );
293
294 addListeners( tc );
295 }
296 }
297
298
299 if( testCasesToRun.size() > 0 )
300 {
301 for( TestCase testCase : testCasesToRun )
302 runTestCase( ( WsdlTestCase )testCase );
303 }
304 else if( testSuite != null )
305 {
306 runSuite( project.getTestSuiteByName( testSuite ) );
307 }
308 else
309 {
310 runProject( project );
311 }
312
313 long timeTaken = ( System.nanoTime() - startTime ) / 1000000;
314
315 if( printReport )
316 {
317 printReport( timeTaken );
318 }
319
320 exportReports( project );
321
322 if( saveAfterRun && !project.isRemote() )
323 {
324 try
325 {
326 project.save();
327 }
328 catch( Throwable t )
329 {
330 log.error( "Failed to save project", t );
331 }
332 }
333
334 if( ( assertions.size() > 0 || failedTests.size() > 0 ) && !ignoreErrors )
335 {
336 throwFailureException();
337 }
338
339 return true;
340 }
341
342 protected void runProject( WsdlProject project )
343 {
344 try
345 {
346 log.info( ( "Running Project [" + project.getName() + "], runType = " + project.getRunType() ) );
347 WsdlProjectRunner runner = project.run( new StringToObjectMap(), false );
348 log.info( "Project [" + project.getName() + "] finished with status [" + runner.getStatus()
349 +"] in " + runner.getTimeTaken() + "ms" );
350 }
351 catch( Exception e )
352 {
353 e.printStackTrace();
354 }
355 }
356
357 protected void initProject( WsdlProject project ) throws Exception
358 {
359 initProjectProperties( project );
360 }
361
362 protected void exportReports( WsdlProject project ) throws Exception
363 {
364 if( junitReport )
365 {
366 exportJUnitReports( reportCollector, getAbsoluteOutputFolder( project ), project );
367 }
368 }
369
370 protected void addListeners( TestCase tc )
371 {
372 tc.addTestRunListener( this );
373 if( junitReport )
374 tc.addTestRunListener( reportCollector );
375 }
376
377 protected void throwFailureException() throws Exception
378 {
379 StringBuffer buf = new StringBuffer();
380
381 for( int c = 0; c < assertions.size(); c++ )
382 {
383 TestAssertion assertion = assertions.get( c );
384 Assertable assertable = assertion.getAssertable();
385 if( assertable instanceof WsdlTestStep )
386 failedTests.remove( ( ( WsdlTestStep )assertable ).getTestCase() );
387
388 buf.append( assertion.getName() + " in [" + assertable.getModelItem().getName() + "] failed;\n" );
389 buf.append( Arrays.toString( assertion.getErrors() ) + "\n" );
390
391 WsdlTestStepResult result = assertionResults.get( assertion );
392 StringWriter stringWriter = new StringWriter();
393 PrintWriter writer = new PrintWriter( stringWriter );
394 result.writeTo( writer );
395 buf.append( stringWriter.toString() );
396 }
397
398 while( !failedTests.isEmpty() )
399 {
400 buf.append( "TestCase [" + failedTests.remove( 0 ).getName() + "] failed without assertions\n" );
401 }
402
403 throw new Exception( buf.toString() );
404 }
405
406 public void exportJUnitReports( JUnitReportCollector collector, String folder, WsdlProject project ) throws Exception
407 {
408 collector.saveReports( folder == null ? "" : folder );
409 }
410
411 public void printReport( long timeTaken )
412 {
413 System.out.println();
414 System.out.println( "SoapUI " + SoapUI.SOAPUI_VERSION + " TestCaseRunner Summary" );
415 System.out.println( "-----------------------------" );
416 System.out.println( "Time Taken: " + timeTaken + "ms" );
417 System.out.println( "Total TestSuites: " + testSuiteCount );
418 System.out.println( "Total TestCases: " + testCaseCount + " (" + failedTests.size() + " failed)" );
419 System.out.println( "Total TestSteps: " + testStepCount );
420 System.out.println( "Total Request Assertions: " + testAssertionCount );
421 System.out.println( "Total Failed Assertions: " + assertions.size() );
422 System.out.println( "Total Exported Results: " + exportCount );
423 }
424
425 /***
426 * Run tests in the specified TestSuite
427 *
428 * @param suite
429 * the TestSuite to run
430 */
431
432 protected void runSuite( WsdlTestSuite suite )
433 {
434 try
435 {
436 log.info( ( "Running TestSuite [" + suite.getName() + "], runType = " + suite.getRunType() ) );
437 WsdlTestSuiteRunner runner = suite.run( new StringToObjectMap(), false );
438 log.info( "TestSuite [" + suite.getName() + "] finished with status [" + runner.getStatus() + "] in " + ( runner.getTimeTaken() ) + "ms" );
439 }
440 catch( Exception e )
441 {
442 e.printStackTrace();
443 }
444 }
445
446 /***
447 * Runs the specified TestCase
448 *
449 * @param testCase
450 * the testcase to run
451 * @param context
452 */
453
454 protected void runTestCase( WsdlTestCase testCase )
455 {
456 try
457 {
458 log.info( "Running TestCase [" + testCase.getName() + "]" );
459 WsdlTestCaseRunner runner = testCase.run( new StringToObjectMap(), false );
460 log.info( "TestCase [" + testCase.getName() + "] finished with status [" + runner.getStatus() + "] in " + ( runner.getTimeTaken() ) + "ms" );
461 }
462 catch( Exception e )
463 {
464 e.printStackTrace();
465 }
466 }
467
468 /***
469 * Sets the testcase to run
470 *
471 * @param testCase
472 * the testcase to run
473 */
474
475 public void setTestCase( String testCase )
476 {
477 this.testCase = testCase;
478 }
479
480 /***
481 * Sets the TestSuite to run. If not set all TestSuites in the specified
482 * project file are run
483 *
484 * @param testSuite
485 * the testSuite to run.
486 */
487
488 public void setTestSuite( String testSuite )
489 {
490 this.testSuite = testSuite;
491 }
492
493 public void beforeRun( TestCaseRunner testRunner, TestCaseRunContext runContext )
494 {
495 log.info( "Running soapUI testcase [" + testRunner.getTestCase().getName() + "]" );
496 }
497
498 public void beforeStep( TestCaseRunner testRunner, TestCaseRunContext runContext, TestStep currentStep )
499 {
500 super.beforeStep( testRunner, runContext, currentStep );
501
502 if( currentStep != null )
503 log.info( "running step [" + currentStep.getName() + "]" );
504 }
505
506 public void afterStep( TestCaseRunner testRunner, TestCaseRunContext runContext, TestStepResult result )
507 {
508 super.afterStep( testRunner, runContext, result );
509 TestStep currentStep = runContext.getCurrentStep();
510
511 if( currentStep instanceof Assertable )
512 {
513 Assertable requestStep = ( Assertable )currentStep;
514 for( int c = 0; c < requestStep.getAssertionCount(); c++ )
515 {
516 TestAssertion assertion = requestStep.getAssertionAt( c );
517 log.info( "Assertion [" + assertion.getName() + "] has status " + assertion.getStatus() );
518 if( assertion.getStatus() == AssertionStatus.FAILED )
519 {
520 for( AssertionError error : assertion.getErrors() )
521 log.error( "ASSERTION FAILED -> " + error.getMessage() );
522
523 assertions.add( assertion );
524 assertionResults.put( assertion, ( WsdlTestStepResult )result );
525 }
526
527 testAssertionCount++ ;
528 }
529 }
530
531 String countPropertyName = currentStep.getName() + " run count";
532 Long count = ( Long )runContext.getProperty( countPropertyName );
533 if( count == null )
534 {
535 count = new Long( 0 );
536 }
537
538 runContext.setProperty( countPropertyName, new Long( count.longValue() + 1 ) );
539
540 if( result.getStatus() == TestStepStatus.FAILED || exportAll )
541 {
542 try
543 {
544 TestCase tc = currentStep.getTestCase();
545 String nameBase = StringUtils.createFileName( tc.getTestSuite().getName(), '_' ) + "-"
546 + StringUtils.createFileName( tc.getName(), '_' ) + "-"
547 + StringUtils.createFileName( currentStep.getName(), '_' ) + "-" + count.longValue() + "-"
548 + result.getStatus();
549
550 String absoluteOutputFolder = getAbsoluteOutputFolder( ModelSupport.getModelItemProject( tc ));
551 String fileName = absoluteOutputFolder + File.separator + nameBase + ".txt";
552
553 if( result.getStatus() == TestStepStatus.FAILED )
554 log.error( currentStep.getName() + " failed, exporting to [" + fileName + "]" );
555
556 PrintWriter writer = new PrintWriter( fileName );
557 result.writeTo( writer );
558 writer.close();
559
560
561 if( result instanceof WsdlTestRequestStepResult )
562 {
563 Attachment[] attachments = ( ( WsdlTestRequestStepResult )result ).getResponseAttachments();
564 if( attachments != null && attachments.length > 0 )
565 {
566 for( int c = 0; c < attachments.length; c++ )
567 {
568 fileName = nameBase + "-attachment-" + ( c + 1 ) + ".";
569
570 Attachment attachment = attachments[c];
571 String contentType = attachment.getContentType();
572 if( !"application/octet-stream".equals( contentType ) && contentType != null
573 && contentType.indexOf( '/' ) != -1 )
574 {
575 fileName += contentType.substring( contentType.lastIndexOf( '/' ) + 1 );
576 }
577 else
578 {
579 fileName += "dat";
580 }
581
582 fileName = absoluteOutputFolder + File.separator + fileName;
583
584 FileOutputStream outFile = new FileOutputStream( fileName );
585 Tools.writeAll( outFile, attachment.getInputStream() );
586 outFile.close();
587 }
588 }
589 }
590
591 exportCount++ ;
592 }
593 catch( Exception e )
594 {
595 log.error( "Error saving failed result: " + e, e );
596 }
597 }
598
599 testStepCount++ ;
600 }
601
602 public void afterRun( TestCaseRunner testRunner, TestCaseRunContext runContext )
603 {
604 log.info( "Finished running soapUI testcase [" + testRunner.getTestCase().getName() + "], time taken: "
605 + testRunner.getTimeTaken() + "ms, status: " + testRunner.getStatus() );
606
607 if( testRunner.getStatus() == Status.FAILED )
608 {
609 failedTests.add( testRunner.getTestCase() );
610 }
611
612 testCaseCount++ ;
613 }
614 }