1
2
3
4
5
6
7
8
9
10
11
12
13 package com.eviware.soapui.tools;
14
15 import com.eviware.soapui.SoapUI;
16 import com.eviware.soapui.impl.wsdl.WsdlProject;
17 import com.eviware.soapui.impl.wsdl.WsdlTestSuite;
18 import com.eviware.soapui.impl.wsdl.testcase.WsdlTestCase;
19 import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestRequestStepResult;
20 import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestStep;
21 import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestStepResult;
22 import com.eviware.soapui.model.iface.Attachment;
23 import com.eviware.soapui.model.project.ProjectFactoryRegistry;
24 import com.eviware.soapui.model.propertyexpansion.DefaultPropertyExpansionContext;
25 import com.eviware.soapui.model.propertyexpansion.PropertyExpansionContext;
26 import com.eviware.soapui.model.testsuite.*;
27 import com.eviware.soapui.model.testsuite.Assertable.AssertionStatus;
28 import com.eviware.soapui.model.testsuite.AssertionError;
29 import com.eviware.soapui.model.testsuite.TestRunner.Status;
30 import com.eviware.soapui.model.testsuite.TestStepResult.TestStepStatus;
31 import com.eviware.soapui.model.testsuite.TestSuite.TestSuiteRunType;
32 import com.eviware.soapui.report.JUnitReportCollector;
33 import com.eviware.soapui.support.StringUtils;
34 import com.eviware.soapui.support.Tools;
35 import com.eviware.soapui.support.types.StringToObjectMap;
36 import org.apache.commons.cli.CommandLine;
37
38 import java.io.File;
39 import java.io.FileOutputStream;
40 import java.io.PrintWriter;
41 import java.io.StringWriter;
42 import java.util.*;
43
44 /***
45 * Standalone test-runner used from maven-plugin, can also be used from
46 * command-line (see xdocs) or directly from other classes.
47 * <p>
48 * For standalone usage, set the project file (with setProjectFile) and other
49 * desired properties before calling run
50 * </p>
51 *
52 * @author Ole.Matzura
53 */
54
55 public class SoapUITestCaseRunner extends AbstractSoapUITestRunner
56 {
57 public static final String TITLE = "soapUI " + SoapUI.SOAPUI_VERSION + " TestCase Runner";
58
59 private String testSuite;
60 private String testCase;
61 private List<TestAssertion> assertions = new ArrayList<TestAssertion>();
62 private Map<TestAssertion, WsdlTestStepResult> assertionResults = new HashMap<TestAssertion, WsdlTestStepResult>();
63 private List<TestCase> runningTests = new ArrayList<TestCase>();
64 private List<TestCase> failedTests = new ArrayList<TestCase>();
65
66 private int testSuiteCount;
67 private int testCaseCount;
68 private int testStepCount;
69 private int testAssertionCount;
70
71 private boolean printReport;
72 private boolean exportAll;
73 private boolean ignoreErrors;
74 private boolean junitReport;
75 private int exportCount;
76 private JUnitReportCollector reportCollector;
77 private String wssPasswordType;
78 private WsdlProject project;
79
80 private String projectPassword;
81
82 /***
83 * Runs the tests in the specified soapUI project file, see soapUI xdocs for
84 * details.
85 *
86 * @param args
87 * @throws Exception
88 */
89
90 public static void main( String[] args ) throws Exception
91 {
92 new SoapUITestCaseRunner().runFromCommandLine( args );
93 }
94
95 protected boolean processCommandLine( CommandLine cmd )
96 {
97 if( cmd.hasOption( "e" ) )
98 setEndpoint( cmd.getOptionValue( "e" ) );
99
100 if( cmd.hasOption( "s" ) )
101 setTestSuite( getCommandLineOptionSubstSpace( cmd, "s" ) );
102
103 if( cmd.hasOption( "c" ) )
104 setTestCase( getCommandLineOptionSubstSpace( cmd, "c" ) );
105
106 if( cmd.hasOption( "u" ) )
107 setUsername( cmd.getOptionValue( "u" ) );
108
109 if( cmd.hasOption( "p" ) )
110 setPassword( cmd.getOptionValue( "p" ) );
111
112 if( cmd.hasOption( "w" ) )
113 setWssPasswordType( cmd.getOptionValue( "w" ) );
114
115 if( cmd.hasOption( "d" ) )
116 setDomain( cmd.getOptionValue( "d" ) );
117
118 if( cmd.hasOption( "h" ) )
119 setHost( cmd.getOptionValue( "h" ) );
120
121 if( cmd.hasOption( "f" ) )
122 setOutputFolder( getCommandLineOptionSubstSpace( cmd, "f" ) );
123
124 if( cmd.hasOption( "t" ) )
125 setSettingsFile( getCommandLineOptionSubstSpace( cmd, "t" ) );
126
127 if( cmd.hasOption( "x" ) )
128 {
129 setProjectPassword( cmd.getOptionValue( "x" ) );
130 }
131
132 if( cmd.hasOption( "v" ) )
133 {
134 setSoapUISettingsPassword( cmd.getOptionValue( "v" ) );
135 }
136
137 if( cmd.hasOption( "D" ) )
138 {
139 setSystemProperties( cmd.getOptionValues( "D" ) );
140 }
141
142 if( cmd.hasOption( "G" ) )
143 {
144 setGlobalProperties( cmd.getOptionValues( "G" ) );
145 }
146
147 setEnableUI( cmd.hasOption( "i" ) );
148 setPrintReport( cmd.hasOption( "r" ) );
149 setExportAll( cmd.hasOption( "a" ) );
150 setJUnitReport( cmd.hasOption( "j" ) );
151
152 return true;
153 }
154
155 public void setProjectPassword( String projectPassword )
156 {
157 this.projectPassword = projectPassword;
158 }
159
160 public String getProjectPassword()
161 {
162 return projectPassword;
163 }
164
165 protected SoapUIOptions initCommandLineOptions()
166 {
167 SoapUIOptions options = new SoapUIOptions( "testrunner" );
168 options.addOption( "e", true, "Sets the endpoint" );
169 options.addOption( "s", true, "Sets the testsuite" );
170 options.addOption( "c", true, "Sets the testcase" );
171 options.addOption( "u", true, "Sets the username" );
172 options.addOption( "p", true, "Sets the password" );
173 options.addOption( "w", true, "Sets the WSS password type, either 'Text' or 'Digest'" );
174 options.addOption( "i", false, "Enables Swing UI for scripts" );
175 options.addOption( "d", true, "Sets the domain" );
176 options.addOption( "h", true, "Sets the host" );
177 options.addOption( "r", false, "Prints a small summary report" );
178 options.addOption( "f", true, "Sets the output folder to export results to" );
179 options.addOption( "j", false, "Sets the output to include JUnit XML reports" );
180 options.addOption( "a", false, "Turns on exporting of all results" );
181 options.addOption( "t", true, "Sets the soapui-settings.xml file to use" );
182 options.addOption( "x", true, "Sets project password for decryption if project is encrypted" );
183 options.addOption( "v", true, "Sets password for soapui-settings.xml file" );
184 options.addOption( "D", true, "Sets system property with name=value");
185 options.addOption( "G", true, "Sets global property with name=value" );
186
187 return options;
188 }
189
190 /***
191 * Add console appender to groovy log
192 */
193
194 public void setExportAll( boolean exportAll )
195 {
196 this.exportAll = exportAll;
197 }
198
199 public void setJUnitReport( boolean junitReport )
200 {
201 this.junitReport = junitReport;
202 if( junitReport )
203 reportCollector = new JUnitReportCollector();
204 }
205
206 public SoapUITestCaseRunner()
207 {
208 super( SoapUITestCaseRunner.TITLE );
209 }
210
211 public SoapUITestCaseRunner( String title )
212 {
213 super( title );
214 }
215
216 /***
217 * Controls if a short test summary should be printed after the test runs
218 *
219 * @param printReport a flag controlling if a summary should be printed
220 */
221
222 public void setPrintReport( boolean printReport )
223 {
224 this.printReport = printReport;
225 }
226
227 public void setIgnoreError( boolean ignoreErrors )
228 {
229 this.ignoreErrors = ignoreErrors;
230 }
231
232 public boolean runRunner() throws Exception
233 {
234 initGroovyLog();
235
236 assertions.clear();
237
238 String projectFile = getProjectFile();
239
240
241 project = (WsdlProject) ProjectFactoryRegistry.getProjectFactory( "wsdl" ).createNew( projectFile, getProjectPassword() );
242
243 if( project.isDisabled() )
244 throw new Exception( "Failed to load soapUI project file [" + projectFile + "]" );
245
246 initProject();
247 ensureOutputFolder( project );
248
249 log.info( "Running soapUI tests in project [" + project.getName() + "]" );
250
251 long startTime = System.nanoTime();
252
253
254
255 for( int c = 0; c < project.getTestSuiteCount(); c++ )
256 {
257 TestSuite suite = project.getTestSuiteAt( c );
258 for( int i = 0; i < suite.getTestCaseCount(); i++ )
259 {
260 TestCase tc = suite.getTestCaseAt( i );
261 addListeners( tc );
262 }
263 }
264
265
266 for( int c = 0; c < project.getTestSuiteCount(); c++ )
267 {
268 WsdlTestSuite ts = project.getTestSuiteAt( c );
269 if( !ts.isDisabled() && ( testSuite == null || ts.getName().equalsIgnoreCase( testSuite ) ) )
270 {
271 runSuite( ts );
272 testSuiteCount++;
273
274
275 if( !runningTests.isEmpty() )
276 log.info( "Waiting for " + runningTests.size() + " tests to finish" );
277
278 while( !runningTests.isEmpty() )
279 {
280 Thread.sleep( 100 );
281 }
282 }
283 }
284
285 long timeTaken = ( System.nanoTime() - startTime ) / 1000000;
286
287 if( printReport )
288 {
289 printReport( timeTaken );
290 }
291
292 exportReports( project );
293
294 if( assertions.size() > 0 || failedTests.size() > 0 && !ignoreErrors )
295 {
296 throwFailureException();
297 }
298
299 return true;
300 }
301
302 protected void initProject() throws Exception
303 {
304 }
305
306 protected void exportReports( WsdlProject project ) throws Exception
307 {
308 if( junitReport )
309 {
310 exportJUnitReports( reportCollector, getAbsoluteOutputFolder( project ) );
311 }
312 }
313
314 protected void addListeners( TestCase tc )
315 {
316 tc.addTestRunListener( this );
317 if( junitReport )
318 tc.addTestRunListener( reportCollector );
319 }
320
321 protected void throwFailureException() throws Exception
322 {
323 StringBuffer buf = new StringBuffer();
324
325 for( int c = 0; c < assertions.size(); c++ )
326 {
327 TestAssertion assertion = assertions.get( c );
328 Assertable assertable = assertion.getAssertable();
329 if( assertable instanceof WsdlTestStep )
330 failedTests.remove( ( (WsdlTestStep) assertable ).getTestCase() );
331
332 buf.append( assertion.getName() + " in [" + assertable.getModelItem().getName() + "] failed;\n" );
333 buf.append( Arrays.toString( assertion.getErrors() ) + "\n" );
334
335 WsdlTestStepResult result = assertionResults.get( assertion );
336 StringWriter stringWriter = new StringWriter();
337 PrintWriter writer = new PrintWriter( stringWriter );
338 result.writeTo( writer );
339 buf.append( stringWriter.toString() );
340 }
341
342 while( !failedTests.isEmpty() )
343 {
344 buf.append( "TestCase [" + failedTests.remove( 0 ).getName() + "] failed without assertions\n" );
345 }
346
347 throw new Exception( buf.toString() );
348 }
349
350 public void exportJUnitReports( JUnitReportCollector collector, String folder ) throws Exception
351 {
352 collector.saveReports( folder == null ? "" : folder );
353 }
354
355 public void printReport( long timeTaken )
356 {
357 System.out.println();
358 System.out.println( "SoapUI " + SoapUI.SOAPUI_VERSION + " TestCaseRunner Summary" );
359 System.out.println( "-----------------------------" );
360 System.out.println( "Time Taken: " + timeTaken + "ms" );
361 System.out.println( "Total TestSuites: " + testSuiteCount );
362 System.out.println( "Total TestCases: " + testCaseCount + " (" + failedTests.size() + " failed)" );
363 System.out.println( "Total TestSteps: " + testStepCount );
364 System.out.println( "Total Request Assertions: " + testAssertionCount );
365 System.out.println( "Total Failed Assertions: " + assertions.size() );
366 System.out.println( "Total Exported Results: " + exportCount );
367 }
368
369 /***
370 * Run tests in the specified TestSuite
371 *
372 * @param suite the TestSuite to run
373 */
374
375 public void runSuite( WsdlTestSuite suite )
376 {
377 log.info( ( "Running soapUI suite [" + suite.getName() + "], runType = " + suite.getRunType() ) );
378 PropertyExpansionContext context = new DefaultPropertyExpansionContext( suite );
379 long start = System.currentTimeMillis();
380
381 try
382 {
383 suite.runSetupScript( context );
384 for( int c = 0; c < suite.getTestCaseCount(); c++ )
385 {
386 WsdlTestCase tc = suite.getTestCaseAt( c );
387
388 String name = tc.getName();
389 if( testCase == null || name.equalsIgnoreCase( testCase ) )
390 {
391 if( !tc.isDisabled() )
392 {
393 runTestCase( tc );
394 }
395 else
396 {
397 log.info( "Skipping disabled testcase [" + name + "]" );
398 }
399 }
400 else
401 {
402 log.info( "Skipping testcase [" + name + "], filter is [" + testCase + "]" );
403 }
404 }
405 }
406 catch( Exception e )
407 {
408 e.printStackTrace();
409 }
410 finally
411 {
412 try
413 {
414 suite.runTearDownScript( context );
415 }
416 catch( Exception e )
417 {
418 e.printStackTrace();
419 }
420 }
421
422 log.info( "soapUI suite [" + suite.getName() + "] finished in " + ( System.currentTimeMillis() - start ) + "ms" );
423 }
424
425 /***
426 * Runs the specified TestCase
427 *
428 * @param testCase the testcase to run
429 */
430
431 private void runTestCase( TestCase testCase )
432 {
433 runningTests.add( testCase );
434 testCase.run( new StringToObjectMap(), testCase.getTestSuite().getRunType() == TestSuiteRunType.PARALLEL );
435 }
436
437 /***
438 * Sets the testcase to run
439 *
440 * @param testCase the testcase to run
441 */
442
443 public void setTestCase( String testCase )
444 {
445 this.testCase = testCase;
446 }
447
448 /***
449 * Sets the TestSuite to run. If not set all TestSuites in the specified
450 * project file are run
451 *
452 * @param testSuite the testSuite to run.
453 */
454
455 public void setTestSuite( String testSuite )
456 {
457 this.testSuite = testSuite;
458 }
459
460 public void beforeRun( TestRunner testRunner, TestRunContext runContext )
461 {
462 log.info( "Running soapUI testcase [" + testRunner.getTestCase().getName() + "]" );
463 }
464
465 public void beforeStep( TestRunner testRunner, TestRunContext runContext )
466 {
467 super.beforeStep( testRunner, runContext );
468
469 TestStep currentStep = runContext.getCurrentStep();
470 log.info( "running step [" + currentStep.getName() + "]" );
471 }
472
473 public void afterStep( TestRunner testRunner, TestRunContext runContext, TestStepResult result )
474 {
475 super.afterStep( testRunner, runContext, result );
476 TestStep currentStep = runContext.getCurrentStep();
477
478 if( currentStep instanceof Assertable )
479 {
480 Assertable requestStep = (Assertable) currentStep;
481 for( int c = 0; c < requestStep.getAssertionCount(); c++ )
482 {
483 TestAssertion assertion = requestStep.getAssertionAt( c );
484 log.info( "Assertion [" + assertion.getName() + "] has status " + assertion.getStatus() );
485 if( assertion.getStatus() == AssertionStatus.FAILED )
486 {
487 for( AssertionError error : assertion.getErrors() )
488 log.error( "ASSERTION FAILED -> " + error.getMessage() );
489
490 assertions.add( assertion );
491 assertionResults.put( assertion, (WsdlTestStepResult) result );
492 }
493
494 testAssertionCount++;
495 }
496 }
497
498 String countPropertyName = currentStep.getName() + " run count";
499 Long count = (Long) runContext.getProperty( countPropertyName );
500 if( count == null )
501 {
502 count = new Long( 0 );
503 }
504
505 runContext.setProperty( countPropertyName, new Long( count.longValue() + 1 ) );
506
507 if( result.getStatus() == TestStepStatus.FAILED || exportAll )
508 {
509 try
510 {
511 TestCase tc = currentStep.getTestCase();
512 String nameBase = StringUtils.createFileName( tc.getTestSuite().getName(), '_' ) + "-"
513 + StringUtils.createFileName( tc.getName(), '_' ) + "-"
514 + StringUtils.createFileName( currentStep.getName(), '_' ) + "-" + count.longValue() + "-"
515 + result.getStatus();
516
517 String absoluteOutputFolder = getAbsoluteOutputFolder( project );
518 String fileName = absoluteOutputFolder + File.separator + nameBase + ".txt";
519
520 if( result.getStatus() == TestStepStatus.FAILED )
521 log.error( currentStep.getName() + " failed, exporting to [" + fileName + "]" );
522
523 PrintWriter writer = new PrintWriter( fileName );
524 result.writeTo( writer );
525 writer.close();
526
527
528 if( result instanceof WsdlTestRequestStepResult )
529 {
530 Attachment[] attachments = ( (WsdlTestRequestStepResult) result ).getResponseAttachments();
531 if( attachments != null && attachments.length > 0 )
532 {
533 for( int c = 0; c < attachments.length; c++ )
534 {
535 fileName = nameBase + "-attachment-" + ( c + 1 ) + ".";
536
537 Attachment attachment = attachments[c];
538 String contentType = attachment.getContentType();
539 if( !"application/octet-stream".equals( contentType ) && contentType != null
540 && contentType.indexOf( '/' ) != -1 )
541 {
542 fileName += contentType.substring( contentType.lastIndexOf( '/' ) + 1 );
543 }
544 else
545 {
546 fileName += "dat";
547 }
548
549 fileName = absoluteOutputFolder + File.separator + fileName;
550
551 FileOutputStream outFile = new FileOutputStream( fileName );
552 Tools.writeAll( outFile, attachment.getInputStream() );
553 outFile.close();
554 }
555 }
556 }
557
558 exportCount++;
559 }
560 catch( Exception e )
561 {
562 log.error( "Error saving failed result: " + e, e );
563 }
564 }
565
566 testStepCount++;
567 }
568
569 public void afterRun( TestRunner testRunner, TestRunContext runContext )
570 {
571 log.info( "Finished running soapUI testcase [" + testRunner.getTestCase().getName() + "], time taken: "
572 + testRunner.getTimeTaken() + "ms, status: " + testRunner.getStatus() );
573
574 if( testRunner.getStatus() == Status.FAILED )
575 {
576 failedTests.add( testRunner.getTestCase() );
577 }
578
579 runningTests.remove( testRunner.getTestCase() );
580
581 testCaseCount++;
582 }
583
584 protected WsdlProject getProject()
585 {
586 return project;
587 }
588 }