1
2
3
4
5
6
7
8
9
10
11
12
13 package com.eviware.soapui.tools;
14
15 import java.io.File;
16 import java.io.FileOutputStream;
17 import java.io.PrintWriter;
18 import java.io.StringWriter;
19 import java.util.ArrayList;
20 import java.util.Arrays;
21 import java.util.HashMap;
22 import java.util.List;
23 import java.util.Map;
24
25 import org.apache.commons.cli.CommandLine;
26
27 import com.eviware.soapui.SoapUI;
28 import com.eviware.soapui.impl.wsdl.WsdlProject;
29 import com.eviware.soapui.impl.wsdl.WsdlTestSuite;
30 import com.eviware.soapui.impl.wsdl.testcase.WsdlTestCase;
31 import com.eviware.soapui.impl.wsdl.teststeps.WsdlMessageAssertion;
32 import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestRequest;
33 import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestRequestStep;
34 import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestRequestStepResult;
35 import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestStepResult;
36 import com.eviware.soapui.model.iface.Attachment;
37 import com.eviware.soapui.model.propertyexpansion.DefaultPropertyExpansionContext;
38 import com.eviware.soapui.model.propertyexpansion.PropertyExpansionContext;
39 import com.eviware.soapui.model.testsuite.AssertionError;
40 import com.eviware.soapui.model.testsuite.TestCase;
41 import com.eviware.soapui.model.testsuite.TestRunContext;
42 import com.eviware.soapui.model.testsuite.TestRunListener;
43 import com.eviware.soapui.model.testsuite.TestRunner;
44 import com.eviware.soapui.model.testsuite.TestStep;
45 import com.eviware.soapui.model.testsuite.TestStepResult;
46 import com.eviware.soapui.model.testsuite.TestSuite;
47 import com.eviware.soapui.model.testsuite.Assertable.AssertionStatus;
48 import com.eviware.soapui.model.testsuite.TestRunner.Status;
49 import com.eviware.soapui.model.testsuite.TestStepResult.TestStepStatus;
50 import com.eviware.soapui.model.testsuite.TestSuite.TestSuiteRunType;
51 import com.eviware.soapui.report.JUnitReportCollector;
52 import com.eviware.soapui.support.Tools;
53 import com.eviware.soapui.support.types.StringToObjectMap;
54
55 /***
56 * Standalone test-runner used from maven-plugin, can also be used from command-line (see xdocs) or
57 * directly from other classes.
58 * <p>
59 * For standalone usage, set the project file (with setProjectFile) and other desired properties before
60 * calling run</p>
61 *
62 * @author Ole.Matzura
63 */
64
65 public class SoapUITestCaseRunner extends AbstractSoapUIRunner implements TestRunListener
66 {
67 public static final String TITLE = "soapUI " + SoapUI.SOAPUI_VERSION + " TestCase Runner";
68
69 private String testSuite;
70 private String testCase;
71 private List<WsdlMessageAssertion> assertions = new ArrayList<WsdlMessageAssertion>();
72 private Map<WsdlMessageAssertion,WsdlTestStepResult> assertionResults = new HashMap<WsdlMessageAssertion,WsdlTestStepResult>();
73 private List<TestCase> runningTests = new ArrayList<TestCase>();
74 private List<TestCase> failedTests = new ArrayList<TestCase>();
75 private String endpoint;
76 private String domain;
77 private String password;
78 private String username;
79 private String host;
80
81 private int testSuiteCount;
82 private int testCaseCount;
83 private int testStepCount;
84 private int testAssertionCount;
85
86 private boolean printReport;
87 private String outputFolder;
88 private boolean exportAll;
89 private boolean junitReport;
90 private int exportCount;
91 private JUnitReportCollector reportCollector;
92 private String wssPasswordType;
93 private WsdlProject project;
94
95 /***
96 * Runs the tests in the specified soapUI project file, see soapUI xdocs for details.
97 *
98 * @param args
99 * @throws Exception
100 */
101
102 @SuppressWarnings("static-access")
103 public static void main( String [] args) throws Exception
104 {
105 new SoapUITestCaseRunner().runFromCommandLine( args );
106 }
107
108 protected boolean processCommandLine( CommandLine cmd )
109 {
110 if( cmd.hasOption( "e"))
111 setEndpoint( cmd.getOptionValue( "e" ) );
112
113 if( cmd.hasOption( "s"))
114 setTestSuite( cmd.getOptionValue( "s") );
115
116 if( cmd.hasOption( "c"))
117 setTestCase( cmd.getOptionValue( "c") );
118
119 if( cmd.hasOption( "u"))
120 setUsername( cmd.getOptionValue( "u") );
121
122 if( cmd.hasOption( "p"))
123 setPassword( cmd.getOptionValue( "p") );
124
125 if( cmd.hasOption( "w"))
126 setWssPasswordType( cmd.getOptionValue( "w") );
127
128 if( cmd.hasOption( "d"))
129 setDomain( cmd.getOptionValue( "d") );
130
131 if( cmd.hasOption( "h"))
132 setHost( cmd.getOptionValue( "h") );
133
134 if( cmd.hasOption( "f"))
135 setOutputFolder( cmd.getOptionValue( "f") );
136
137 if( cmd.hasOption( "t"))
138 setSettingsFile( cmd.getOptionValue( "t" ));
139
140 setEnableUI( cmd.hasOption( "i") );
141 setPrintReport( cmd.hasOption( "r" ) );
142 setExportAll( cmd.hasOption( "a" ) );
143 setJUnitReport( cmd.hasOption( "j" ) );
144
145 return true;
146 }
147
148 protected SoapUIOptions initCommandLineOptions()
149 {
150 SoapUIOptions options = new SoapUIOptions( "testrunner" );
151 options.addOption( "e", true, "Sets the endpoint" );
152 options.addOption( "s", true, "Sets the testsuite" );
153 options.addOption( "c", true, "Sets the testcase" );
154 options.addOption( "u", true, "Sets the username" );
155 options.addOption( "p", true, "Sets the password" );
156 options.addOption( "w", true, "Sets the WSS password type, either 'Text' or 'Digest'" );
157 options.addOption( "i", false, "Enables Swing UI for scripts" );
158 options.addOption( "d", true, "Sets the domain" );
159 options.addOption( "h", true, "Sets the host" );
160 options.addOption( "r", false, "Prints a small summary report" );
161 options.addOption( "f", true, "Sets the output folder to export results to" );
162 options.addOption( "j", false, "Sets the output to include JUnit XML reports" );
163 options.addOption( "a", false, "Turns on exporting of all results" );
164 options.addOption( "t", true, "Sets the soapui-settings.xml file to use" );
165 return options;
166 }
167
168 /***
169 * Add console appender to groovy log
170 */
171
172
173 public void setExportAll(boolean exportAll)
174 {
175 this.exportAll = exportAll;
176 }
177
178 public void setJUnitReport(boolean junitReport)
179 {
180 this.junitReport = junitReport;
181 if (junitReport)
182 reportCollector = new JUnitReportCollector();
183 }
184
185 public void setOutputFolder(String outputFolder)
186 {
187 this.outputFolder = outputFolder;
188 }
189
190 public SoapUITestCaseRunner()
191 {
192 super( SoapUITestCaseRunner.TITLE );
193 }
194
195 public SoapUITestCaseRunner( String title )
196 {
197 super( title );
198 }
199
200 /***
201 * Controls if a short test summary should be printed after the test runs
202 *
203 * @param printReport a flag controlling if a summary should be printed
204 */
205
206 public void setPrintReport(boolean printReport)
207 {
208 this.printReport = printReport;
209 }
210
211 /***
212 * Sets the host to use by all test-requests, the existing endpoint port and path will be used
213 *
214 * @param host the host to use by all requests
215 */
216
217 public void setHost(String host)
218 {
219 this.host = host;
220 }
221
222 /***
223 * Sets the domain to use for any authentications
224 *
225 * @param domain the domain to use for any authentications
226 */
227
228 public void setDomain(String domain)
229 {
230 this.domain = domain;
231 }
232
233 /***
234 * Sets the password to use for any authentications
235 *
236 * @param domain the password to use for any authentications
237 */
238
239 public void setPassword(String password)
240 {
241 this.password = password;
242 }
243
244 /***
245 * Sets the WSS password-type to use for any authentications. Setting this will result
246 * in the addition of WS-Security UsernamePassword tokens to any outgoing request containing
247 * the specified username and password.
248 *
249 * @param wssPasswordType the wss-password type to use, either 'Text' or 'Digest'
250 */
251
252 public void setWssPasswordType( String wssPasswordType )
253 {
254 this.wssPasswordType = wssPasswordType;
255 }
256
257 /***
258 * Sets the username to use for any authentications
259 *
260 * @param domain the username to use for any authentications
261 */
262
263 public void setUsername(String username)
264 {
265 this.username = username;
266 }
267
268 /***
269 * Runs the testcases as configured with setXXX methods
270 *
271 * @throws Exception thrown if any tests fail
272 */
273
274 public boolean runRunner() throws Exception
275 {
276 initGroovyLog();
277
278 if( outputFolder != null )
279 {
280
281 File folder = new File( outputFolder );
282 if( !folder.exists())
283 folder.mkdirs();
284 }
285
286 assertions.clear();
287
288 String projectFile = getProjectFile();
289
290 project = new WsdlProject( projectFile );
291 if( project.isDisabled() )
292 throw new Exception( "Failed to load soapUI project file [" + projectFile + "]" );
293
294 log.info( "Running soapUI tests in project [" + project.getName() + "]" );
295
296 long startTime = System.nanoTime();
297
298
299 for( int c = 0; c < project.getTestSuiteCount(); c++ )
300 {
301 TestSuite suite = project.getTestSuiteAt( c );
302 for( int i = 0; i < suite.getTestCaseCount(); i++ )
303 {
304 suite.getTestCaseAt( i ).addTestRunListener( this );
305 if (junitReport)
306 suite.getTestCaseAt( i ).addTestRunListener(reportCollector);
307 }
308 }
309
310
311 for( int c = 0; c < project.getTestSuiteCount(); c++ )
312 {
313 WsdlTestSuite ts = project.getTestSuiteAt( c );
314 if( !ts.isDisabled() && (testSuite == null || ts.getName().equalsIgnoreCase( testSuite )))
315 {
316 runSuite( ts);
317 testSuiteCount++;
318
319
320 if( !runningTests.isEmpty() )
321 log.info( "Waiting for " + runningTests.size() + " tests to finish" );
322
323 while( !runningTests.isEmpty() )
324 {
325 Thread.sleep( 100 );
326 }
327 }
328 }
329
330 long timeTaken = (System.nanoTime()-startTime)/1000000;
331
332 if( printReport )
333 {
334 printReport( timeTaken );
335 }
336
337 if (junitReport) {
338 exportJUnitReports( reportCollector, outputFolder );
339 }
340
341 if( assertions.size() > 0 || failedTests.size() > 0 )
342 {
343 throwFailureException();
344 }
345
346 return true;
347 }
348
349 protected void throwFailureException() throws Exception
350 {
351 StringBuffer buf = new StringBuffer();
352
353 for( int c = 0; c < assertions.size(); c++ )
354 {
355 WsdlMessageAssertion assertion = assertions.get( c );
356 WsdlTestRequest testRequest = ((WsdlTestRequest)assertion.getAssertable());
357 failedTests.remove( testRequest.getTestCase() );
358
359 buf.append( assertion.getName() + " in [" + testRequest.getName() + "] failed;\n" );
360 buf.append( Arrays.toString( assertion.getErrors() ) + "\n" );
361
362 WsdlTestStepResult result = assertionResults.get( assertion );
363 StringWriter stringWriter = new StringWriter();
364 PrintWriter writer = new PrintWriter( stringWriter );
365 result.writeTo( writer );
366 buf.append( stringWriter.toString() );
367 }
368
369 while( !failedTests.isEmpty() )
370 {
371 buf.append( "TestCase [" + failedTests.remove( 0 ).getName() + "] failed without assertions\n" );
372 }
373
374 throw new Exception( buf.toString() );
375 }
376
377 public void exportJUnitReports( JUnitReportCollector collector, String folder ) throws Exception
378 {
379 collector.saveReports(folder == null ? "" : folder);
380 }
381
382 public void printReport( long timeTaken )
383 {
384 System.out.println();
385 System.out.println( "SoapUI " + SoapUI.SOAPUI_VERSION + " TestCaseRunner Summary" );
386 System.out.println( "-----------------------------" );
387 System.out.println( "Time Taken: " + timeTaken + "ms" );
388 System.out.println( "Total TestSuites: " + testSuiteCount );
389 System.out.println( "Total TestCases: " + testCaseCount + " (" + failedTests.size() + " failed)");
390 System.out.println( "Total TestSteps: " + testStepCount );
391 System.out.println( "Total Request Assertions: " + testAssertionCount );
392 System.out.println( "Total Failed Assertions: " + assertions.size() );
393 System.out.println( "Total Exported Results: " + exportCount );
394 }
395
396 /***
397 * Run tests in the specified TestSuite
398 *
399 * @param suite the TestSuite to run
400 */
401
402 public void runSuite(WsdlTestSuite suite)
403 {
404 log.info(( "Running soapUI suite [" + suite.getName() + "], runType = " + suite.getRunType()));
405 PropertyExpansionContext context = new DefaultPropertyExpansionContext( suite );
406 long start = System.currentTimeMillis();
407
408 try
409 {
410 suite.runSetupScript( context );
411 for( int c = 0; c < suite.getTestCaseCount(); c++ )
412 {
413 WsdlTestCase tc = suite.getTestCaseAt( c );
414
415 String name = tc.getName();
416 if( testCase == null || name.equalsIgnoreCase( testCase ))
417 {
418 if( !tc.isDisabled() )
419 {
420 runTestCase( tc);
421 }
422 else
423 {
424 log.info( "Skipping disabled testcase [" + name + "]");
425 }
426 }
427 else
428 {
429 log.info( "Skipping testcase [" + name + "], filter is [" + testCase + "]");
430 }
431 }
432 }
433 catch( Exception e )
434 {
435 e.printStackTrace();
436 }
437 finally
438 {
439 try
440 {
441 suite.runTearDownScript( context );
442 }
443 catch( Exception e )
444 {
445 e.printStackTrace();
446 }
447 }
448
449 log.info( "soapUI suite [" + suite.getName() + "] finished in " + (System.currentTimeMillis()-start) + "ms" );
450 }
451
452 /***
453 * Runs the specified TestCase
454 *
455 * @param testCase the testcase to run
456 */
457
458 private void runTestCase(TestCase testCase)
459 {
460 runningTests.add( testCase );
461 testCase.run( new StringToObjectMap(), testCase.getTestSuite().getRunType() == TestSuiteRunType.PARALLEL );
462 }
463
464 /***
465 * Sets the testcase to run
466 *
467 * @param testCase the testcase to run
468 */
469
470 public void setTestCase(String testCase)
471 {
472 this.testCase = testCase;
473 }
474
475 /***
476 * Sets the endpoint to use for all test requests
477 *
478 * @param endpoint the endpoint to use for all test requests
479 */
480
481 public void setEndpoint(String endpoint)
482 {
483 this.endpoint = endpoint.trim();
484 }
485
486 /***
487 * Sets the TestSuite to run. If not set all TestSuites in the specified project file are run
488 *
489 * @param testSuite the testSuite to run.
490 */
491
492 public void setTestSuite(String testSuite)
493 {
494 this.testSuite = testSuite;
495 }
496
497 public void beforeRun(TestRunner testRunner, TestRunContext runContext)
498 {
499 log.info( "Running soapUI testcase [" + testRunner.getTestCase().getName() + "]");
500 }
501
502 public void beforeStep(TestRunner testRunner, TestRunContext runContext)
503 {
504 TestStep currentStep = runContext.getCurrentStep();
505 log.info( "running step [" + currentStep.getName() + "]" );
506
507 if( currentStep instanceof WsdlTestRequestStep )
508 {
509 WsdlTestRequestStep requestStep = (WsdlTestRequestStep) currentStep;
510 if( endpoint != null && endpoint.length() > 0 )
511 {
512 requestStep.getTestRequest().setEndpoint( endpoint );
513 }
514
515 if( host != null && host.length() > 0 )
516 {
517 try
518 {
519 String ep = Tools.replaceHost( requestStep.getTestRequest().getEndpoint(), host );
520 requestStep.getTestRequest().setEndpoint( ep );
521 }
522 catch (Exception e)
523 {
524 log.error( "Failed to set host on endpoint", e );
525 }
526 }
527
528 if( username != null && username.length() > 0 )
529 {
530 requestStep.getTestRequest().setUsername( username );
531 }
532
533 if( password != null && password.length() > 0 )
534 {
535 requestStep.getTestRequest().setPassword( password );
536 }
537
538 if( domain != null && domain.length() > 0 )
539 {
540 requestStep.getTestRequest().setDomain( domain );
541 }
542
543 if( wssPasswordType != null && wssPasswordType.length() > 0 )
544 {
545 requestStep.getTestRequest().setWssPasswordType( wssPasswordType.equals( "Digest" ) ?
546 WsdlTestRequest.PW_TYPE_DIGEST : WsdlTestRequest.PW_TYPE_TEXT );
547 }
548 }
549 }
550
551 public void afterStep(TestRunner testRunner, TestRunContext runContext, TestStepResult result )
552 {
553 TestStep currentStep = runContext.getCurrentStep();
554
555 if( currentStep instanceof WsdlTestRequestStep )
556 {
557 WsdlTestRequestStep requestStep = (WsdlTestRequestStep) currentStep;
558 for( int c = 0; c < requestStep.getAssertionCount(); c++ )
559 {
560 WsdlMessageAssertion assertion = requestStep.getAssertionAt( c );
561 log.info( "Assertion [" + assertion.getName() + "] has status " + assertion.getStatus());
562 if( assertion.getStatus() == AssertionStatus.FAILED )
563 {
564 for( AssertionError error : assertion.getErrors())
565 log.error( "ASSERTION FAILED -> " + error.getMessage());
566
567 assertions.add( assertion );
568 assertionResults.put( assertion, ( WsdlTestStepResult ) result );
569 }
570
571 testAssertionCount++;
572 }
573 }
574
575 String countPropertyName = currentStep.getName() + " run count";
576 Long count = (Long) runContext.getProperty( countPropertyName );
577 if( count == null )
578 {
579 count = new Long( 0 );
580 }
581
582 runContext.setProperty( countPropertyName, new Long( count.longValue()+1 ) );
583
584 if( result.getStatus() == TestStepStatus.FAILED || exportAll )
585 {
586 try
587 {
588 String nameBase = currentStep.getTestCase().getTestSuite().getName() + "-"
589 + currentStep.getTestCase().getName() + "-" + currentStep.getName() + "-" +
590 count.longValue() + "-" + result.getStatus();
591
592 String fileName = nameBase + ".txt";
593 if( outputFolder != null )
594 {
595 fileName = outputFolder + File.separator + fileName;
596 }
597
598 if( result.getStatus() == TestStepStatus.FAILED )
599 log.error( currentStep.getName() + " failed, exporting to [" + fileName + "]" );
600
601 PrintWriter writer = new PrintWriter(fileName);
602 result.writeTo(writer);
603 writer.close();
604
605
606 if( result instanceof WsdlTestRequestStepResult )
607 {
608 Attachment [] attachments = ((WsdlTestRequestStepResult)result).getResponseAttachments();
609 if( attachments != null && attachments.length > 0 )
610 {
611 for( int c = 0; c < attachments.length; c++ )
612 {
613 fileName = nameBase + "-attachment-" + (c+1) + ".";
614
615 Attachment attachment = attachments[c];
616 String contentType = attachment.getContentType();
617 if( !"application/octet-stream".equals( contentType ) && contentType != null && contentType.indexOf( '/' ) != -1 )
618 {
619 fileName += contentType.substring( contentType.lastIndexOf( '/' )+1 );
620 }
621 else
622 {
623 fileName += "dat";
624 }
625
626 if( outputFolder != null )
627 {
628 fileName = outputFolder + File.separator + fileName;
629 }
630
631 FileOutputStream outFile = new FileOutputStream( fileName );
632 Tools.writeAll( outFile, attachment.getInputStream() );
633 outFile.close();
634 }
635 }
636 }
637
638 exportCount++;
639 }
640 catch (Exception e)
641 {
642 log.error( "Error saving failed result: " + e, e );
643 }
644 }
645
646 testStepCount++;
647 }
648
649 public void afterRun(TestRunner testRunner, TestRunContext runContext)
650 {
651 log.info( "Finished running soapUI testcase [" + testRunner.getTestCase().getName() + "], time taken: " +
652 testRunner.getTimeTaken() + "ms, status: " + testRunner.getStatus() );
653
654 if( testRunner.getStatus() == Status.FAILED )
655 {
656 failedTests.add( testRunner.getTestCase() );
657 }
658
659 runningTests.remove( testRunner.getTestCase() );
660
661 testCaseCount++;
662 }
663
664 protected WsdlProject getProject()
665 {
666 return project;
667 }
668 }