1
2
3
4
5
6
7
8
9
10
11
12
13 package com.eviware.soapui.tools;
14
15 import java.io.File;
16 import java.io.OutputStreamWriter;
17 import java.io.PrintWriter;
18 import java.io.StringWriter;
19 import java.util.ArrayList;
20 import java.util.Arrays;
21 import java.util.HashMap;
22 import java.util.List;
23 import java.util.Map;
24
25 import org.apache.commons.cli.CommandLine;
26 import org.apache.commons.cli.CommandLineParser;
27 import org.apache.commons.cli.HelpFormatter;
28 import org.apache.commons.cli.Options;
29 import org.apache.commons.cli.PosixParser;
30 import org.apache.log4j.ConsoleAppender;
31 import org.apache.log4j.Logger;
32 import org.apache.log4j.PatternLayout;
33
34 import com.eviware.soapui.SoapUI;
35 import com.eviware.soapui.impl.wsdl.WsdlProject;
36 import com.eviware.soapui.impl.wsdl.panels.support.assertions.Assertable.AssertionStatus;
37 import com.eviware.soapui.impl.wsdl.teststeps.WsdlMessageAssertion;
38 import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestRequest;
39 import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestRequestStep;
40 import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestStepResult;
41 import com.eviware.soapui.model.support.PropertiesMap;
42 import com.eviware.soapui.model.testsuite.TestCase;
43 import com.eviware.soapui.model.testsuite.TestRunContext;
44 import com.eviware.soapui.model.testsuite.TestRunListener;
45 import com.eviware.soapui.model.testsuite.TestRunner;
46 import com.eviware.soapui.model.testsuite.TestStep;
47 import com.eviware.soapui.model.testsuite.TestStepResult;
48 import com.eviware.soapui.model.testsuite.TestSuite;
49 import com.eviware.soapui.model.testsuite.TestRunner.Status;
50 import com.eviware.soapui.model.testsuite.TestStepResult.TestStepStatus;
51 import com.eviware.soapui.model.testsuite.TestSuite.TestSuiteRunType;
52 import com.eviware.soapui.monitor.TestMonitor;
53 import com.eviware.soapui.report.JUnitReportCollector;
54
55 /***
56 * Standalone test-runner used from maven-plugin, can also be used from command-line (see xdocs) or
57 * directly from other classes.
58 * <p>
59 * For standalone usage, set the project file (with setProjectFile) and other desired properties before
60 * calling run</p>
61 *
62 * @author Ole.Matzura
63 */
64
65 public class SoapUITestCaseRunner implements TestRunListener
66 {
67 private final static Logger log = Logger.getLogger( SoapUITestCaseRunner.class );
68
69 private String projectFile;
70 private String testSuite;
71 private String testCase;
72 private List<WsdlMessageAssertion> assertions = new ArrayList<WsdlMessageAssertion>();
73 private Map<WsdlMessageAssertion,WsdlTestStepResult> assertionResults = new HashMap<WsdlMessageAssertion,WsdlTestStepResult>();
74 private List<TestCase> runningTests = new ArrayList<TestCase>();
75 private List<TestCase> failedTests = new ArrayList<TestCase>();
76 private String endpoint;
77 private String domain;
78 private String password;
79 private String username;
80 private String host;
81
82 private int testSuiteCount;
83 private int testCaseCount;
84 private int testStepCount;
85 private int testAssertionCount;
86
87 private boolean printReport;
88 private String outputFolder;
89 private boolean exportAll;
90 private boolean junitReport;
91 private int exportCount;
92 private JUnitReportCollector reportCollector;
93 private String wssPasswordType;
94
95 public static String TITLE = "soapUI " + SoapUI.SOAPUI_VERSION + " TestCase Runner";
96
97 /***
98 * Runs the tests in the specified soapUI project file, see soapUI xdocs for details.
99 *
100 * @param args
101 * @throws Exception
102 */
103
104 @SuppressWarnings("static-access")
105 public static void main( String [] args) throws Exception
106 {
107 System.out.println( TITLE );
108 SoapUI.initSoapUILog();
109 SoapUITestCaseRunner runner = new SoapUITestCaseRunner();
110
111 Options options = new Options();
112 options.addOption( "e", true, "Sets the endpoint" );
113 options.addOption( "s", true, "Sets the testsuite" );
114 options.addOption( "c", true, "Sets the testcase" );
115 options.addOption( "u", true, "Sets the username" );
116 options.addOption( "p", true, "Sets the password" );
117 options.addOption( "w", true, "Sets the WSS password type, either 'Text' or 'Digest'" );
118 options.addOption( "d", true, "Sets the domain" );
119 options.addOption( "h", true, "Sets the host" );
120 options.addOption( "r", false, "Prints a small summary report" );
121 options.addOption( "f", true, "Sets the output folder to export results to" );
122 options.addOption( "j", false, "Sets the output to include JUnit XML reports" );
123 options.addOption( "a", false, "Turns on exporting of all results" );
124 options.addOption( "t", true, "Sets the soapui-settings.xml file to use" );
125
126 CommandLineParser parser = new PosixParser();
127 CommandLine cmd = parser.parse( options, args);
128
129 String[] args2 = cmd.getArgs();
130 if( args2.length != 1 )
131 {
132 HelpFormatter formatter = new HelpFormatter();
133 formatter.printHelp( "testcaserunner [options] <soapui-project-file>", options );
134
135 System.err.println( "Missing soapUI project file.." );
136 return;
137 }
138
139 runner.setProjectFile( args2[0] );
140
141 if( cmd.hasOption( "e"))
142 runner.setEndpoint( cmd.getOptionValue( "e" ) );
143
144 if( cmd.hasOption( "s"))
145 runner.setTestSuite( cmd.getOptionValue( "s") );
146
147 if( cmd.hasOption( "c"))
148 runner.setTestCase( cmd.getOptionValue( "c") );
149
150 if( cmd.hasOption( "u"))
151 runner.setUsername( cmd.getOptionValue( "u") );
152
153 if( cmd.hasOption( "p"))
154 runner.setPassword( cmd.getOptionValue( "p") );
155
156 if( cmd.hasOption( "w"))
157 runner.setWssPasswordType( cmd.getOptionValue( "w") );
158
159 if( cmd.hasOption( "d"))
160 runner.setDomain( cmd.getOptionValue( "d") );
161
162 if( cmd.hasOption( "h"))
163 runner.setHost( cmd.getOptionValue( "h") );
164
165 if( cmd.hasOption( "f"))
166 runner.setOutputFolder( cmd.getOptionValue( "f") );
167
168 if( cmd.hasOption( "t"))
169 SoapUI.initSettings( cmd.getOptionValue( "t" ));
170
171 runner.setPrintReport( cmd.hasOption( "r" ) );
172 runner.setExportAll( cmd.hasOption( "a" ) );
173 runner.setJUnitReport( cmd.hasOption( "j" ) );
174
175 try
176 {
177 SoapUI.loadExtLibs();
178 initGroovyLogger();
179
180 runner.run();
181 System.exit( 0 );
182 }
183 catch (Exception e)
184 {
185 log.error( e.toString() );
186 e.printStackTrace();
187 System.exit( 1 );
188 }
189 }
190
191 /***
192 * Add console appender to groovy log
193 */
194
195 public static void initGroovyLogger()
196 {
197 Logger logger = Logger.getLogger( "groovy.log" );
198
199 ConsoleAppender appender = new ConsoleAppender();
200 appender.setWriter( new OutputStreamWriter( System.out ));
201 appender.setLayout( new PatternLayout( "%d{ABSOLUTE} %-5p [%c{1}] %m%n") );
202 logger.addAppender( appender);
203 }
204
205 public void setExportAll(boolean exportAll)
206 {
207 this.exportAll = exportAll;
208 }
209
210 public void setJUnitReport(boolean junitReport)
211 {
212 this.junitReport = junitReport;
213 if (junitReport)
214 reportCollector = new JUnitReportCollector();
215 }
216
217 public void setOutputFolder(String outputFolder)
218 {
219 this.outputFolder = outputFolder;
220 }
221
222 public SoapUITestCaseRunner()
223 {
224 SoapUI.setTestMonitor( new TestMonitor() );
225 }
226
227 /***
228 * Controls if a short test summary should be printed after the test runs
229 *
230 * @param printReport a flag controlling if a summary should be printed
231 */
232
233 public void setPrintReport(boolean printReport)
234 {
235 this.printReport = printReport;
236 }
237
238 /***
239 * Sets the host to use by all test-requests, the existing endpoint port and path will be used
240 *
241 * @param host the host to use by all requests
242 */
243
244 public void setHost(String host)
245 {
246 this.host = host;
247 }
248
249 /***
250 * Sets the domain to use for any authentications
251 *
252 * @param domain the domain to use for any authentications
253 */
254
255 public void setDomain(String domain)
256 {
257 log.info( "Setting domain to [" + domain + "]" );
258 this.domain = domain;
259 }
260
261 /***
262 * Sets the password to use for any authentications
263 *
264 * @param domain the password to use for any authentications
265 */
266
267 public void setPassword(String password)
268 {
269 log.info( "Setting password to [" + password + "]" );
270 this.password = password;
271 }
272
273 /***
274 * Sets the WSS password-type to use for any authentications. Setting this will result
275 * in the addition of WS-Security UsernamePassword tokens to any outgoing request containing
276 * the specified username and password.
277 *
278 * @param wssPasswordType the wss-password type to use, either 'Text' or 'Digest'
279 */
280
281 public void setWssPasswordType( String wssPasswordType )
282 {
283 this.wssPasswordType = wssPasswordType;
284 }
285
286 /***
287 * Sets the username to use for any authentications
288 *
289 * @param domain the username to use for any authentications
290 */
291
292 public void setUsername(String username)
293 {
294 log.info( "Setting username to [" + username + "]" );
295 this.username = username;
296 }
297
298 /***
299 * Runs the testcases as configured with setXXX methods
300 *
301 * @throws Exception thrown if any tests fail
302 */
303
304 public void run() throws Exception
305 {
306 assertions.clear();
307
308 if( !new File( projectFile ).exists() )
309 throw new Exception( "soapUI project file [" + projectFile + "] not found" );
310
311 WsdlProject project = new WsdlProject( projectFile, null );
312 log.info( "Running soapUI tests in project [" + project.getName() + "]" );
313
314 long startTime = System.nanoTime();
315
316
317 for( int c = 0; c < project.getTestSuiteCount(); c++ )
318 {
319 TestSuite suite = project.getTestSuiteAt( c );
320 for( int i = 0; i < suite.getTestCaseCount(); i++ )
321 {
322 suite.getTestCaseAt( i ).addTestRunListener( this );
323 if (junitReport)
324 suite.getTestCaseAt( i ).addTestRunListener(reportCollector);
325 }
326 }
327
328
329 for( int c = 0; c < project.getTestSuiteCount(); c++ )
330 {
331 if( testSuite == null || project.getTestSuiteAt( c ).getName().equalsIgnoreCase( testSuite ))
332 {
333 runSuite( project.getTestSuiteAt( c ));
334 testSuiteCount++;
335
336
337 if( !runningTests.isEmpty() )
338 log.info( "Waiting for " + runningTests.size() + " tests to finish" );
339
340 while( !runningTests.isEmpty() )
341 {
342 Thread.sleep( 100 );
343 }
344 }
345 }
346
347 long timeTaken = (System.nanoTime()-startTime)/1000000;
348
349 if( printReport )
350 {
351 System.out.println();
352 System.out.println( "SoapUI " + SoapUI.SOAPUI_VERSION + " TestCaseRunner Summary" );
353 System.out.println( "-----------------------------" );
354 System.out.println( "Time Taken: " + timeTaken + "ms" );
355 System.out.println( "Total TestSuites: " + testSuiteCount );
356 System.out.println( "Total TestCases: " + testCaseCount + " (" + failedTests.size() + " failed)");
357 System.out.println( "Total TestSteps: " + testStepCount );
358 System.out.println( "Total Request Assertions: " + testAssertionCount );
359 System.out.println( "Total Failed Assertions: " + assertions.size() );
360 System.out.println( "Total Exported Results: " + exportCount );
361 }
362
363 if (junitReport) {
364 reportCollector.saveReports(outputFolder == null ? "" : outputFolder);
365 }
366
367 if( assertions.size() > 0 || failedTests.size() > 0 )
368 {
369 StringBuffer buf = new StringBuffer();
370
371 for( int c = 0; c < assertions.size(); c++ )
372 {
373 WsdlMessageAssertion assertion = assertions.get( c );
374 WsdlTestRequest testRequest = ((WsdlTestRequest)assertion.getAssertable());
375 failedTests.remove( testRequest.getTestCase() );
376
377 buf.append( assertion.getName() + " in [" + testRequest.getName() + "] failed;\n" );
378 buf.append( Arrays.toString( assertion.getErrors() ) + "\n" );
379
380 WsdlTestStepResult result = assertionResults.get( assertion );
381 StringWriter stringWriter = new StringWriter();
382 PrintWriter writer = new PrintWriter( stringWriter );
383 result.writeTo( writer );
384 buf.append( stringWriter.toString() );
385 }
386
387 while( !failedTests.isEmpty() )
388 {
389 buf.append( "TestCase [" + failedTests.remove( 0 ).getName() + "] failed without assertions\n" );
390 }
391
392 throw new Exception( buf.toString() );
393 }
394 }
395
396 /***
397 * Run tests in the specified TestSuite
398 *
399 * @param suite the TestSuite to run
400 */
401
402 public void runSuite(TestSuite suite)
403 {
404 log.info(( "Running soapUI suite [" + suite.getName() + "], runType = " + suite.getRunType()));
405 long start = System.currentTimeMillis();
406 for( int c = 0; c < suite.getTestCaseCount(); c++ )
407 {
408 String name = suite.getTestCaseAt( c ).getName();
409 if( testCase == null || name.equalsIgnoreCase( testCase ))
410 {
411 runTestCase( suite.getTestCaseAt( c ));
412 }
413 else
414 {
415 log.info( "Skipping testcase [" + name + "], filter is [" + testCase + "]");
416 }
417 }
418 log.info( "soapUI suite [" + suite.getName() + "] finished in " + (System.currentTimeMillis()-start) + "ms" );
419 }
420
421 /***
422 * Runs the specified TestCase
423 *
424 * @param testCase the testcase to run
425 */
426
427 private void runTestCase(TestCase testCase)
428 {
429 runningTests.add( testCase );
430 testCase.run( PropertiesMap.EMPTY_MAP, testCase.getTestSuite().getRunType() == TestSuiteRunType.PARALLEL );
431 }
432
433 /***
434 * Sets the soapUI project file containing the tests to run
435 *
436 * @param projectFile the soapUI project file containing the tests to run
437 */
438
439 public void setProjectFile(String projectFile)
440 {
441 log.info( "setting projectFile to [" + projectFile + "]" );
442 this.projectFile = projectFile;
443 }
444
445 /***
446 * Sets the testcase to run
447 *
448 * @param testCase the testcase to run
449 */
450
451 public void setTestCase(String testCase)
452 {
453 log.info( "setting testCase to [" + testCase + "]" );
454 this.testCase = testCase;
455 }
456
457 /***
458 * Sets the endpoint to use for all test requests
459 *
460 * @param endpoint the endpoint to use for all test requests
461 */
462
463 public void setEndpoint(String endpoint)
464 {
465 log.info( "setting test endpoint to [" + endpoint+ "]" );
466 this.endpoint = endpoint.trim();
467 }
468
469 /***
470 * Sets the TestSuite to run. If not set all TestSuites in the specified project file are run
471 *
472 * @param testSuite the testSuite to run.
473 */
474
475 public void setTestSuite(String testSuite)
476 {
477 log.info( "setting testSuite to [" + testSuite + "]" );
478 this.testSuite = testSuite;
479 }
480
481 public void beforeRun(TestRunner testRunner, TestRunContext runContext)
482 {
483 log.info( "Running soapUI testcase [" + testRunner.getTestCase().getName() + "]");
484 }
485
486 public void beforeStep(TestRunner testRunner, TestRunContext runContext)
487 {
488 TestStep currentStep = runContext.getCurrentStep();
489 log.info( "running step [" + currentStep.getName() + "]" );
490
491 if( currentStep instanceof WsdlTestRequestStep )
492 {
493 WsdlTestRequestStep requestStep = (WsdlTestRequestStep) currentStep;
494 if( endpoint != null && endpoint.length() > 0 )
495 {
496 requestStep.getTestRequest().setEndpoint( endpoint );
497 }
498
499 if( host != null && host.length() > 0 )
500 {
501 try
502 {
503 String ep = replaceHost( requestStep.getTestRequest().getEndpoint(), host );
504 requestStep.getTestRequest().setEndpoint( ep );
505 }
506 catch (Exception e)
507 {
508 log.error( "Failed to set host on endpoint", e );
509 }
510 }
511
512 if( username != null && username.length() > 0 )
513 {
514 requestStep.getTestRequest().setUsername( username );
515 }
516
517 if( password != null && password.length() > 0 )
518 {
519 requestStep.getTestRequest().setPassword( password );
520 }
521
522 if( domain != null && domain.length() > 0 )
523 {
524 requestStep.getTestRequest().setDomain( domain );
525 }
526
527 if( wssPasswordType != null && wssPasswordType.length() > 0 )
528 {
529 requestStep.getTestRequest().setWssPasswordType( wssPasswordType.equals( "Digest" ) ?
530 WsdlTestRequest.PW_TYPE_DIGEST : WsdlTestRequest.PW_TYPE_TEXT );
531 }
532 }
533 }
534
535 public void afterStep(TestRunner testRunner, TestRunContext runContext, TestStepResult result )
536 {
537 TestStep currentStep = runContext.getCurrentStep();
538
539 if( currentStep instanceof WsdlTestRequestStep )
540 {
541 WsdlTestRequestStep requestStep = (WsdlTestRequestStep) currentStep;
542 for( int c = 0; c < requestStep.getAssertionCount(); c++ )
543 {
544 WsdlMessageAssertion assertion = requestStep.getAssertionAt( c );
545 log.info( "Assertion [" + assertion.getName() + "] has status " + assertion.getStatus());
546 if( assertion.getStatus() == AssertionStatus.FAILED )
547 {
548 log.info( "ASSERTION FAILED -> " + assertion.getErrors());
549 assertions.add( assertion );
550 assertionResults.put( assertion, ( WsdlTestStepResult ) result );
551 }
552
553 testAssertionCount++;
554 }
555 }
556
557 String countPropertyName = currentStep.getName() + " run count";
558 Long count = (Long) runContext.getProperty( countPropertyName );
559 if( count == null )
560 {
561 count = new Long( 0 );
562 }
563
564 runContext.setProperty( countPropertyName, new Long( count.longValue()+1 ) );
565
566 if( result.getStatus() == TestStepStatus.FAILED || exportAll )
567 {
568 try
569 {
570 String fileName = currentStep.getTestCase().getTestSuite().getName() + "-"
571 + currentStep.getTestCase().getName() + "-" + currentStep.getName() + "-" +
572 count.longValue() + "-" + result.getStatus() + ".txt";;
573
574 if( outputFolder != null )
575 {
576
577 File folder = new File( outputFolder );
578 if( !folder.exists())
579 folder.mkdirs();
580
581 fileName = outputFolder + File.separator + fileName;
582 }
583
584 if( result.getStatus() == TestStepStatus.FAILED )
585 log.error( currentStep.getName() + " failed, exporting to [" + fileName + "]" );
586
587 PrintWriter writer = new PrintWriter(fileName);
588 result.writeTo(writer);
589 writer.close();
590
591 exportCount++;
592 }
593 catch (Exception e)
594 {
595 log.error( "Error saving failed result: " + e, e );
596 }
597 }
598
599 testStepCount++;
600 }
601
602 public void afterRun(TestRunner testRunner, TestRunContext runContext)
603 {
604 log.info( "Finished running soapUI testcase [" + testRunner.getTestCase().getName() + "], time taken: " +
605 testRunner.getTimeTaken() + "ms, status: " + testRunner.getStatus() );
606
607 if( testRunner.getStatus() == Status.FAILED )
608 {
609 failedTests.add( testRunner.getTestCase() );
610 }
611
612 runningTests.remove( testRunner.getTestCase() );
613
614 testCaseCount++;
615 }
616
617 /***
618 * Replaces the host part of the specified endpoint with the specified host
619 *
620 * @param endpoint the endpoint to modify
621 * @param host the host to set
622 * @return the modified endpoint
623 */
624
625 public static String replaceHost(String endpoint, String host)
626 {
627 int ix1 = endpoint.indexOf( "://" );
628 if( ix1 < 0 )
629 return endpoint;
630
631 int ix2 = endpoint.indexOf( ":", ix1+3 );
632 if( ix2 == -1 || host.indexOf( ":") > 0 )
633 {
634 ix2 = endpoint.indexOf( "/", ix1+3 );
635 if( ix2 == ix1+3 )
636 ix2 = -1;
637 }
638
639 return endpoint.substring( 0, ix1 )+ "://" + host + (ix2 == -1 ? "" : endpoint.substring(ix2));
640 }
641 }