1
2
3
4
5
6
7
8
9
10
11
12
13 package com.eviware.soapui.tools;
14
15 import java.io.File;
16 import java.io.FileOutputStream;
17 import java.io.PrintWriter;
18 import java.io.StringWriter;
19 import java.util.ArrayList;
20 import java.util.Arrays;
21 import java.util.HashMap;
22 import java.util.List;
23 import java.util.Map;
24
25 import org.apache.commons.cli.CommandLine;
26
27 import com.eviware.soapui.SoapUI;
28 import com.eviware.soapui.impl.wsdl.WsdlProject;
29 import com.eviware.soapui.impl.wsdl.WsdlTestSuite;
30 import com.eviware.soapui.impl.wsdl.testcase.WsdlTestCase;
31 import com.eviware.soapui.impl.wsdl.teststeps.WsdlMessageAssertion;
32 import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestRequest;
33 import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestRequestStep;
34 import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestRequestStepResult;
35 import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestStep;
36 import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestStepResult;
37 import com.eviware.soapui.model.iface.Attachment;
38 import com.eviware.soapui.model.propertyexpansion.DefaultPropertyExpansionContext;
39 import com.eviware.soapui.model.propertyexpansion.PropertyExpansionContext;
40 import com.eviware.soapui.model.testsuite.Assertable;
41 import com.eviware.soapui.model.testsuite.AssertionError;
42 import com.eviware.soapui.model.testsuite.TestCase;
43 import com.eviware.soapui.model.testsuite.TestRunContext;
44 import com.eviware.soapui.model.testsuite.TestRunListener;
45 import com.eviware.soapui.model.testsuite.TestRunner;
46 import com.eviware.soapui.model.testsuite.TestStep;
47 import com.eviware.soapui.model.testsuite.TestStepResult;
48 import com.eviware.soapui.model.testsuite.TestSuite;
49 import com.eviware.soapui.model.testsuite.Assertable.AssertionStatus;
50 import com.eviware.soapui.model.testsuite.TestRunner.Status;
51 import com.eviware.soapui.model.testsuite.TestStepResult.TestStepStatus;
52 import com.eviware.soapui.model.testsuite.TestSuite.TestSuiteRunType;
53 import com.eviware.soapui.report.JUnitReportCollector;
54 import com.eviware.soapui.support.Tools;
55 import com.eviware.soapui.support.types.StringToObjectMap;
56
57 /***
58 * Standalone test-runner used from maven-plugin, can also be used from command-line (see xdocs) or
59 * directly from other classes.
60 * <p>
61 * For standalone usage, set the project file (with setProjectFile) and other desired properties before
62 * calling run</p>
63 *
64 * @author Ole.Matzura
65 */
66
67 public class SoapUITestCaseRunner extends AbstractSoapUIRunner implements TestRunListener
68 {
69 public static final String TITLE = "soapUI " + SoapUI.SOAPUI_VERSION + " TestCase Runner";
70
71 private String testSuite;
72 private String testCase;
73 private List<WsdlMessageAssertion> assertions = new ArrayList<WsdlMessageAssertion>();
74 private Map<WsdlMessageAssertion,WsdlTestStepResult> assertionResults = new HashMap<WsdlMessageAssertion,WsdlTestStepResult>();
75 private List<TestCase> runningTests = new ArrayList<TestCase>();
76 private List<TestCase> failedTests = new ArrayList<TestCase>();
77 private String endpoint;
78 private String domain;
79 private String password;
80 private String username;
81 private String host;
82
83 private int testSuiteCount;
84 private int testCaseCount;
85 private int testStepCount;
86 private int testAssertionCount;
87
88 private boolean printReport;
89 private String outputFolder;
90 private boolean exportAll;
91 private boolean junitReport;
92 private int exportCount;
93 private JUnitReportCollector reportCollector;
94 private String wssPasswordType;
95 private WsdlProject project;
96
97 /***
98 * Runs the tests in the specified soapUI project file, see soapUI xdocs for details.
99 *
100 * @param args
101 * @throws Exception
102 */
103
104 @SuppressWarnings("static-access")
105 public static void main( String [] args) throws Exception
106 {
107 new SoapUITestCaseRunner().runFromCommandLine( args );
108 }
109
110 protected boolean processCommandLine( CommandLine cmd )
111 {
112 if( cmd.hasOption( "e"))
113 setEndpoint( cmd.getOptionValue( "e" ) );
114
115 if( cmd.hasOption( "s"))
116 setTestSuite( getCommandLineOptionSubstSpace( cmd, "s") );
117
118 if( cmd.hasOption( "c"))
119 setTestCase( getCommandLineOptionSubstSpace( cmd, "c") );
120
121 if( cmd.hasOption( "u"))
122 setUsername( cmd.getOptionValue( "u") );
123
124 if( cmd.hasOption( "p"))
125 setPassword( cmd.getOptionValue( "p") );
126
127 if( cmd.hasOption( "w"))
128 setWssPasswordType( cmd.getOptionValue( "w") );
129
130 if( cmd.hasOption( "d"))
131 setDomain( cmd.getOptionValue( "d") );
132
133 if( cmd.hasOption( "h"))
134 setHost( cmd.getOptionValue( "h") );
135
136 if( cmd.hasOption( "f"))
137 setOutputFolder( getCommandLineOptionSubstSpace( cmd, "f") );
138
139 if( cmd.hasOption( "t"))
140 setSettingsFile( getCommandLineOptionSubstSpace( cmd, "t" ));
141
142 setEnableUI( cmd.hasOption( "i") );
143 setPrintReport( cmd.hasOption( "r" ) );
144 setExportAll( cmd.hasOption( "a" ) );
145 setJUnitReport( cmd.hasOption( "j" ) );
146
147 return true;
148 }
149
150 protected SoapUIOptions initCommandLineOptions()
151 {
152 SoapUIOptions options = new SoapUIOptions( "testrunner" );
153 options.addOption( "e", true, "Sets the endpoint" );
154 options.addOption( "s", true, "Sets the testsuite" );
155 options.addOption( "c", true, "Sets the testcase" );
156 options.addOption( "u", true, "Sets the username" );
157 options.addOption( "p", true, "Sets the password" );
158 options.addOption( "w", true, "Sets the WSS password type, either 'Text' or 'Digest'" );
159 options.addOption( "i", false, "Enables Swing UI for scripts" );
160 options.addOption( "d", true, "Sets the domain" );
161 options.addOption( "h", true, "Sets the host" );
162 options.addOption( "r", false, "Prints a small summary report" );
163 options.addOption( "f", true, "Sets the output folder to export results to" );
164 options.addOption( "j", false, "Sets the output to include JUnit XML reports" );
165 options.addOption( "a", false, "Turns on exporting of all results" );
166 options.addOption( "t", true, "Sets the soapui-settings.xml file to use" );
167 return options;
168 }
169
170 /***
171 * Add console appender to groovy log
172 */
173
174
175 public void setExportAll(boolean exportAll)
176 {
177 this.exportAll = exportAll;
178 }
179
180 public void setJUnitReport(boolean junitReport)
181 {
182 this.junitReport = junitReport;
183 if (junitReport)
184 reportCollector = new JUnitReportCollector();
185 }
186
187 public void setOutputFolder(String outputFolder)
188 {
189 this.outputFolder = outputFolder;
190 }
191
192 public SoapUITestCaseRunner()
193 {
194 super( SoapUITestCaseRunner.TITLE );
195 }
196
197 public SoapUITestCaseRunner( String title )
198 {
199 super( title );
200 }
201
202 /***
203 * Controls if a short test summary should be printed after the test runs
204 *
205 * @param printReport a flag controlling if a summary should be printed
206 */
207
208 public void setPrintReport(boolean printReport)
209 {
210 this.printReport = printReport;
211 }
212
213 /***
214 * Sets the host to use by all test-requests, the existing endpoint port and path will be used
215 *
216 * @param host the host to use by all requests
217 */
218
219 public void setHost(String host)
220 {
221 this.host = host;
222 }
223
224 /***
225 * Sets the domain to use for any authentications
226 *
227 * @param domain the domain to use for any authentications
228 */
229
230 public void setDomain(String domain)
231 {
232 this.domain = domain;
233 }
234
235 /***
236 * Sets the password to use for any authentications
237 *
238 * @param domain the password to use for any authentications
239 */
240
241 public void setPassword(String password)
242 {
243 this.password = password;
244 }
245
246 /***
247 * Sets the WSS password-type to use for any authentications. Setting this will result
248 * in the addition of WS-Security UsernamePassword tokens to any outgoing request containing
249 * the specified username and password.
250 *
251 * @param wssPasswordType the wss-password type to use, either 'Text' or 'Digest'
252 */
253
254 public void setWssPasswordType( String wssPasswordType )
255 {
256 this.wssPasswordType = wssPasswordType;
257 }
258
259 /***
260 * Sets the username to use for any authentications
261 *
262 * @param domain the username to use for any authentications
263 */
264
265 public void setUsername(String username)
266 {
267 this.username = username;
268 }
269
270 /***
271 * Runs the testcases as configured with setXXX methods
272 *
273 * @throws Exception thrown if any tests fail
274 */
275
276 public boolean runRunner() throws Exception
277 {
278 initGroovyLog();
279
280 if( outputFolder != null )
281 {
282
283 File folder = new File( outputFolder );
284 if( !folder.exists())
285 folder.mkdirs();
286 }
287
288 assertions.clear();
289
290 String projectFile = getProjectFile();
291
292 project = new WsdlProject( projectFile );
293 if( project.isDisabled() )
294 throw new Exception( "Failed to load soapUI project file [" + projectFile + "]" );
295
296 log.info( "Running soapUI tests in project [" + project.getName() + "]" );
297
298 long startTime = System.nanoTime();
299
300
301 for( int c = 0; c < project.getTestSuiteCount(); c++ )
302 {
303 TestSuite suite = project.getTestSuiteAt( c );
304 for( int i = 0; i < suite.getTestCaseCount(); i++ )
305 {
306 suite.getTestCaseAt( i ).addTestRunListener( this );
307 if (junitReport)
308 suite.getTestCaseAt( i ).addTestRunListener(reportCollector);
309 }
310 }
311
312
313 for( int c = 0; c < project.getTestSuiteCount(); c++ )
314 {
315 WsdlTestSuite ts = project.getTestSuiteAt( c );
316 if( !ts.isDisabled() && (testSuite == null || ts.getName().equalsIgnoreCase( testSuite )))
317 {
318 runSuite( ts);
319 testSuiteCount++;
320
321
322 if( !runningTests.isEmpty() )
323 log.info( "Waiting for " + runningTests.size() + " tests to finish" );
324
325 while( !runningTests.isEmpty() )
326 {
327 Thread.sleep( 100 );
328 }
329 }
330 }
331
332 long timeTaken = (System.nanoTime()-startTime)/1000000;
333
334 if( printReport )
335 {
336 printReport( timeTaken );
337 }
338
339 if (junitReport) {
340 exportJUnitReports( reportCollector, outputFolder );
341 }
342
343 if( assertions.size() > 0 || failedTests.size() > 0 )
344 {
345 throwFailureException();
346 }
347
348 return true;
349 }
350
351 protected void throwFailureException() throws Exception
352 {
353 StringBuffer buf = new StringBuffer();
354
355 for( int c = 0; c < assertions.size(); c++ )
356 {
357 WsdlMessageAssertion assertion = assertions.get( c );
358 Assertable assertable = assertion.getAssertable();
359 if( assertable instanceof WsdlTestStep)
360 failedTests.remove( ((WsdlTestStep)assertable).getTestCase() );
361
362 buf.append( assertion.getName() + " in [" + assertable.getModelItem().getName() + "] failed;\n" );
363 buf.append( Arrays.toString( assertion.getErrors() ) + "\n" );
364
365 WsdlTestStepResult result = assertionResults.get( assertion );
366 StringWriter stringWriter = new StringWriter();
367 PrintWriter writer = new PrintWriter( stringWriter );
368 result.writeTo( writer );
369 buf.append( stringWriter.toString() );
370 }
371
372 while( !failedTests.isEmpty() )
373 {
374 buf.append( "TestCase [" + failedTests.remove( 0 ).getName() + "] failed without assertions\n" );
375 }
376
377 throw new Exception( buf.toString() );
378 }
379
380 public void exportJUnitReports( JUnitReportCollector collector, String folder ) throws Exception
381 {
382 collector.saveReports(folder == null ? "" : folder);
383 }
384
385 public void printReport( long timeTaken )
386 {
387 System.out.println();
388 System.out.println( "SoapUI " + SoapUI.SOAPUI_VERSION + " TestCaseRunner Summary" );
389 System.out.println( "-----------------------------" );
390 System.out.println( "Time Taken: " + timeTaken + "ms" );
391 System.out.println( "Total TestSuites: " + testSuiteCount );
392 System.out.println( "Total TestCases: " + testCaseCount + " (" + failedTests.size() + " failed)");
393 System.out.println( "Total TestSteps: " + testStepCount );
394 System.out.println( "Total Request Assertions: " + testAssertionCount );
395 System.out.println( "Total Failed Assertions: " + assertions.size() );
396 System.out.println( "Total Exported Results: " + exportCount );
397 }
398
399 /***
400 * Run tests in the specified TestSuite
401 *
402 * @param suite the TestSuite to run
403 */
404
405 public void runSuite(WsdlTestSuite suite)
406 {
407 log.info(( "Running soapUI suite [" + suite.getName() + "], runType = " + suite.getRunType()));
408 PropertyExpansionContext context = new DefaultPropertyExpansionContext( suite );
409 long start = System.currentTimeMillis();
410
411 try
412 {
413 suite.runSetupScript( context );
414 for( int c = 0; c < suite.getTestCaseCount(); c++ )
415 {
416 WsdlTestCase tc = suite.getTestCaseAt( c );
417
418 String name = tc.getName();
419 if( testCase == null || name.equalsIgnoreCase( testCase ))
420 {
421 if( !tc.isDisabled() )
422 {
423 runTestCase( tc);
424 }
425 else
426 {
427 log.info( "Skipping disabled testcase [" + name + "]");
428 }
429 }
430 else
431 {
432 log.info( "Skipping testcase [" + name + "], filter is [" + testCase + "]");
433 }
434 }
435 }
436 catch( Exception e )
437 {
438 e.printStackTrace();
439 }
440 finally
441 {
442 try
443 {
444 suite.runTearDownScript( context );
445 }
446 catch( Exception e )
447 {
448 e.printStackTrace();
449 }
450 }
451
452 log.info( "soapUI suite [" + suite.getName() + "] finished in " + (System.currentTimeMillis()-start) + "ms" );
453 }
454
455 /***
456 * Runs the specified TestCase
457 *
458 * @param testCase the testcase to run
459 */
460
461 private void runTestCase(TestCase testCase)
462 {
463 runningTests.add( testCase );
464 testCase.run( new StringToObjectMap(), testCase.getTestSuite().getRunType() == TestSuiteRunType.PARALLEL );
465 }
466
467 /***
468 * Sets the testcase to run
469 *
470 * @param testCase the testcase to run
471 */
472
473 public void setTestCase(String testCase)
474 {
475 this.testCase = testCase;
476 }
477
478 /***
479 * Sets the endpoint to use for all test requests
480 *
481 * @param endpoint the endpoint to use for all test requests
482 */
483
484 public void setEndpoint(String endpoint)
485 {
486 this.endpoint = endpoint.trim();
487 }
488
489 /***
490 * Sets the TestSuite to run. If not set all TestSuites in the specified project file are run
491 *
492 * @param testSuite the testSuite to run.
493 */
494
495 public void setTestSuite(String testSuite)
496 {
497 this.testSuite = testSuite;
498 }
499
500 public void beforeRun(TestRunner testRunner, TestRunContext runContext)
501 {
502 log.info( "Running soapUI testcase [" + testRunner.getTestCase().getName() + "]");
503 }
504
505 public void beforeStep(TestRunner testRunner, TestRunContext runContext)
506 {
507 TestStep currentStep = runContext.getCurrentStep();
508 log.info( "running step [" + currentStep.getName() + "]" );
509
510 if( currentStep instanceof WsdlTestRequestStep )
511 {
512 WsdlTestRequestStep requestStep = (WsdlTestRequestStep) currentStep;
513 if( endpoint != null && endpoint.length() > 0 )
514 {
515 requestStep.getTestRequest().setEndpoint( endpoint );
516 }
517
518 if( host != null && host.length() > 0 )
519 {
520 try
521 {
522 String ep = Tools.replaceHost( requestStep.getTestRequest().getEndpoint(), host );
523 requestStep.getTestRequest().setEndpoint( ep );
524 }
525 catch (Exception e)
526 {
527 log.error( "Failed to set host on endpoint", e );
528 }
529 }
530
531 if( username != null && username.length() > 0 )
532 {
533 requestStep.getTestRequest().setUsername( username );
534 }
535
536 if( password != null && password.length() > 0 )
537 {
538 requestStep.getTestRequest().setPassword( password );
539 }
540
541 if( domain != null && domain.length() > 0 )
542 {
543 requestStep.getTestRequest().setDomain( domain );
544 }
545
546 if( wssPasswordType != null && wssPasswordType.length() > 0 )
547 {
548 requestStep.getTestRequest().setWssPasswordType( wssPasswordType.equals( "Digest" ) ?
549 WsdlTestRequest.PW_TYPE_DIGEST : WsdlTestRequest.PW_TYPE_TEXT );
550 }
551 }
552 }
553
554 public void afterStep(TestRunner testRunner, TestRunContext runContext, TestStepResult result )
555 {
556 TestStep currentStep = runContext.getCurrentStep();
557
558 if( currentStep instanceof WsdlTestRequestStep )
559 {
560 WsdlTestRequestStep requestStep = (WsdlTestRequestStep) currentStep;
561 for( int c = 0; c < requestStep.getAssertionCount(); c++ )
562 {
563 WsdlMessageAssertion assertion = requestStep.getAssertionAt( c );
564 log.info( "Assertion [" + assertion.getName() + "] has status " + assertion.getStatus());
565 if( assertion.getStatus() == AssertionStatus.FAILED )
566 {
567 for( AssertionError error : assertion.getErrors())
568 log.error( "ASSERTION FAILED -> " + error.getMessage());
569
570 assertions.add( assertion );
571 assertionResults.put( assertion, ( WsdlTestStepResult ) result );
572 }
573
574 testAssertionCount++;
575 }
576 }
577
578 String countPropertyName = currentStep.getName() + " run count";
579 Long count = (Long) runContext.getProperty( countPropertyName );
580 if( count == null )
581 {
582 count = new Long( 0 );
583 }
584
585 runContext.setProperty( countPropertyName, new Long( count.longValue()+1 ) );
586
587 if( result.getStatus() == TestStepStatus.FAILED || exportAll )
588 {
589 try
590 {
591 String nameBase = currentStep.getTestCase().getTestSuite().getName() + "-"
592 + currentStep.getTestCase().getName() + "-" + currentStep.getName() + "-" +
593 count.longValue() + "-" + result.getStatus();
594
595 String fileName = nameBase + ".txt";
596 if( outputFolder != null )
597 {
598 fileName = outputFolder + File.separator + fileName;
599 }
600
601 if( result.getStatus() == TestStepStatus.FAILED )
602 log.error( currentStep.getName() + " failed, exporting to [" + fileName + "]" );
603
604 PrintWriter writer = new PrintWriter(fileName);
605 result.writeTo(writer);
606 writer.close();
607
608
609 if( result instanceof WsdlTestRequestStepResult )
610 {
611 Attachment [] attachments = ((WsdlTestRequestStepResult)result).getResponseAttachments();
612 if( attachments != null && attachments.length > 0 )
613 {
614 for( int c = 0; c < attachments.length; c++ )
615 {
616 fileName = nameBase + "-attachment-" + (c+1) + ".";
617
618 Attachment attachment = attachments[c];
619 String contentType = attachment.getContentType();
620 if( !"application/octet-stream".equals( contentType ) && contentType != null && contentType.indexOf( '/' ) != -1 )
621 {
622 fileName += contentType.substring( contentType.lastIndexOf( '/' )+1 );
623 }
624 else
625 {
626 fileName += "dat";
627 }
628
629 if( outputFolder != null )
630 {
631 fileName = outputFolder + File.separator + fileName;
632 }
633
634 FileOutputStream outFile = new FileOutputStream( fileName );
635 Tools.writeAll( outFile, attachment.getInputStream() );
636 outFile.close();
637 }
638 }
639 }
640
641 exportCount++;
642 }
643 catch (Exception e)
644 {
645 log.error( "Error saving failed result: " + e, e );
646 }
647 }
648
649 testStepCount++;
650 }
651
652 public void afterRun(TestRunner testRunner, TestRunContext runContext)
653 {
654 log.info( "Finished running soapUI testcase [" + testRunner.getTestCase().getName() + "], time taken: " +
655 testRunner.getTimeTaken() + "ms, status: " + testRunner.getStatus() );
656
657 if( testRunner.getStatus() == Status.FAILED )
658 {
659 failedTests.add( testRunner.getTestCase() );
660 }
661
662 runningTests.remove( testRunner.getTestCase() );
663
664 testCaseCount++;
665 }
666
667 protected WsdlProject getProject()
668 {
669 return project;
670 }
671 }