1
2
3
4
5
6
7
8
9
10
11
12
13 package com.eviware.soapui.tools;
14
15 import java.io.File;
16 import java.io.OutputStreamWriter;
17 import java.io.PrintWriter;
18 import java.io.StringWriter;
19 import java.util.ArrayList;
20 import java.util.Arrays;
21 import java.util.HashMap;
22 import java.util.List;
23 import java.util.Map;
24
25 import org.apache.commons.cli.CommandLine;
26 import org.apache.commons.cli.CommandLineParser;
27 import org.apache.commons.cli.HelpFormatter;
28 import org.apache.commons.cli.Options;
29 import org.apache.commons.cli.PosixParser;
30 import org.apache.log4j.ConsoleAppender;
31 import org.apache.log4j.Logger;
32 import org.apache.log4j.PatternLayout;
33
34 import com.eviware.soapui.SoapUI;
35 import com.eviware.soapui.impl.wsdl.WsdlProject;
36 import com.eviware.soapui.impl.wsdl.panels.support.assertions.Assertable.AssertionStatus;
37 import com.eviware.soapui.impl.wsdl.teststeps.WsdlMessageAssertion;
38 import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestRequest;
39 import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestRequestStep;
40 import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestStepResult;
41 import com.eviware.soapui.model.support.PropertiesMap;
42 import com.eviware.soapui.model.testsuite.TestCase;
43 import com.eviware.soapui.model.testsuite.TestRunContext;
44 import com.eviware.soapui.model.testsuite.TestRunListener;
45 import com.eviware.soapui.model.testsuite.TestRunner;
46 import com.eviware.soapui.model.testsuite.TestStep;
47 import com.eviware.soapui.model.testsuite.TestStepResult;
48 import com.eviware.soapui.model.testsuite.TestSuite;
49 import com.eviware.soapui.model.testsuite.TestRunner.Status;
50 import com.eviware.soapui.model.testsuite.TestStepResult.TestStepStatus;
51 import com.eviware.soapui.model.testsuite.TestSuite.TestSuiteRunType;
52 import com.eviware.soapui.monitor.TestMonitor;
53 import com.eviware.soapui.report.JUnitReportCollector;
54
55 /***
56 * Standalone test-runner used from maven-plugin, can also be used from command-line (see xdocs) or
57 * directly from other classes.
58 * <p>
59 * For standalone usage, set the project file (with setProjectFile) and other desired properties before
60 * calling run</p>
61 *
62 * @author Ole.Matzura
63 */
64
65 public class SoapUITestCaseRunner implements TestRunListener
66 {
67 private final static Logger log = Logger.getLogger( SoapUITestCaseRunner.class );
68
69 private String projectFile;
70 private String testSuite;
71 private String testCase;
72 private List<WsdlMessageAssertion> assertions = new ArrayList<WsdlMessageAssertion>();
73 private Map<WsdlMessageAssertion,WsdlTestStepResult> assertionResults = new HashMap<WsdlMessageAssertion,WsdlTestStepResult>();
74 private List<TestCase> runningTests = new ArrayList<TestCase>();
75 private List<TestCase> failedTests = new ArrayList<TestCase>();
76 private String endpoint;
77 private String domain;
78 private String password;
79 private String username;
80 private String host;
81
82 private int testSuiteCount;
83 private int testCaseCount;
84 private int testStepCount;
85 private int testAssertionCount;
86
87 private boolean printReport;
88 private String outputFolder;
89 private boolean exportAll;
90 private boolean junitReport;
91 private int exportCount;
92 private JUnitReportCollector reportCollector;
93 private String wssPasswordType;
94
95 public static String TITLE = "soapUI " + SoapUI.SOAPUI_VERSION + " TestCase Runner";
96
97 /***
98 * Runs the tests in the specified soapUI project file, see soapUI xdocs for details.
99 *
100 * @param args
101 * @throws Exception
102 */
103
104 @SuppressWarnings("static-access")
105 public static void main( String [] args) throws Exception
106 {
107 System.out.println( TITLE );
108 SoapUITestCaseRunner runner = new SoapUITestCaseRunner();
109
110 Options options = new Options();
111 options.addOption( "e", true, "Sets the endpoint" );
112 options.addOption( "s", true, "Sets the testsuite" );
113 options.addOption( "c", true, "Sets the testcase" );
114 options.addOption( "u", true, "Sets the username" );
115 options.addOption( "p", true, "Sets the password" );
116 options.addOption( "w", true, "Sets the WSS password type, either 'Text' or 'Digest'" );
117 options.addOption( "d", true, "Sets the domain" );
118 options.addOption( "h", true, "Sets the host" );
119 options.addOption( "r", false, "Prints a small summary report" );
120 options.addOption( "f", true, "Sets the output folder to export results to" );
121 options.addOption( "j", false, "Sets the output to include JUnit XML reports" );
122 options.addOption( "a", false, "Turns on exporting of all results" );
123 options.addOption( "t", true, "Sets the soapui-settings.xml file to use" );
124
125 CommandLineParser parser = new PosixParser();
126 CommandLine cmd = parser.parse( options, args);
127
128 String[] args2 = cmd.getArgs();
129 if( args2.length != 1 )
130 {
131 HelpFormatter formatter = new HelpFormatter();
132 formatter.printHelp( "testcaserunner [options] <soapui-project-file>", options );
133
134 System.err.println( "Missing soapUI project file.." );
135 return;
136 }
137
138 runner.setProjectFile( args2[0] );
139
140 if( cmd.hasOption( "e"))
141 runner.setEndpoint( cmd.getOptionValue( "e" ) );
142
143 if( cmd.hasOption( "s"))
144 runner.setTestSuite( cmd.getOptionValue( "s") );
145
146 if( cmd.hasOption( "c"))
147 runner.setTestCase( cmd.getOptionValue( "c") );
148
149 if( cmd.hasOption( "u"))
150 runner.setUsername( cmd.getOptionValue( "u") );
151
152 if( cmd.hasOption( "p"))
153 runner.setPassword( cmd.getOptionValue( "p") );
154
155 if( cmd.hasOption( "w"))
156 runner.setWssPasswordType( cmd.getOptionValue( "w") );
157
158 if( cmd.hasOption( "d"))
159 runner.setDomain( cmd.getOptionValue( "d") );
160
161 if( cmd.hasOption( "h"))
162 runner.setHost( cmd.getOptionValue( "h") );
163
164 if( cmd.hasOption( "f"))
165 runner.setOutputFolder( cmd.getOptionValue( "f") );
166
167 if( cmd.hasOption( "t"))
168 SoapUI.initSettings( cmd.getOptionValue( "t" ));
169
170 runner.setPrintReport( cmd.hasOption( "r" ) );
171 runner.setExportAll( cmd.hasOption( "a" ) );
172 runner.setJUnitReport( cmd.hasOption( "j" ) );
173
174 try
175 {
176 SoapUI.loadExtLibs();
177 initGroovyLogger();
178
179 runner.run();
180 System.exit( 0 );
181 }
182 catch (Exception e)
183 {
184 log.error( e.toString() );
185 e.printStackTrace();
186 System.exit( 1 );
187 }
188 }
189
190 /***
191 * Add console appender to groovy log
192 */
193
194 public static void initGroovyLogger()
195 {
196 Logger logger = Logger.getLogger( "groovy.log" );
197
198 ConsoleAppender appender = new ConsoleAppender();
199 appender.setWriter( new OutputStreamWriter( System.out ));
200 appender.setLayout( new PatternLayout( "%d{ABSOLUTE} %-5p [%c{1}] %m%n") );
201 logger.addAppender( appender);
202 }
203
204 public void setExportAll(boolean exportAll)
205 {
206 this.exportAll = exportAll;
207 }
208
209 public void setJUnitReport(boolean junitReport)
210 {
211 this.junitReport = junitReport;
212 if (junitReport)
213 reportCollector = new JUnitReportCollector();
214 }
215
216 public void setOutputFolder(String outputFolder)
217 {
218 this.outputFolder = outputFolder;
219 }
220
221 public SoapUITestCaseRunner()
222 {
223 SoapUI.setTestMonitor( new TestMonitor() );
224 }
225
226 /***
227 * Controls if a short test summary should be printed after the test runs
228 *
229 * @param printReport a flag controlling if a summary should be printed
230 */
231
232 public void setPrintReport(boolean printReport)
233 {
234 this.printReport = printReport;
235 }
236
237 /***
238 * Sets the host to use by all test-requests, the existing endpoint port and path will be used
239 *
240 * @param host the host to use by all requests
241 */
242
243 public void setHost(String host)
244 {
245 this.host = host;
246 }
247
248 /***
249 * Sets the domain to use for any authentications
250 *
251 * @param domain the domain to use for any authentications
252 */
253
254 public void setDomain(String domain)
255 {
256 log.info( "Setting domain to [" + domain + "]" );
257 this.domain = domain;
258 }
259
260 /***
261 * Sets the password to use for any authentications
262 *
263 * @param domain the password to use for any authentications
264 */
265
266 public void setPassword(String password)
267 {
268 log.info( "Setting password to [" + password + "]" );
269 this.password = password;
270 }
271
272 /***
273 * Sets the WSS password-type to use for any authentications. Setting this will result
274 * in the addition of WS-Security UsernamePassword tokens to any outgoing request containing
275 * the specified username and password.
276 *
277 * @param wssPasswordType the wss-password type to use, either 'Text' or 'Digest'
278 */
279
280 public void setWssPasswordType( String wssPasswordType )
281 {
282 this.wssPasswordType = wssPasswordType;
283 }
284
285 /***
286 * Sets the username to use for any authentications
287 *
288 * @param domain the username to use for any authentications
289 */
290
291 public void setUsername(String username)
292 {
293 log.info( "Setting username to [" + username + "]" );
294 this.username = username;
295 }
296
297 /***
298 * Runs the testcases as configured with setXXX methods
299 *
300 * @throws Exception thrown if any tests fail
301 */
302
303 public void run() throws Exception
304 {
305 assertions.clear();
306
307 if( !new File( projectFile ).exists() )
308 throw new Exception( "soapUI project file [" + projectFile + "] not found" );
309
310 WsdlProject project = new WsdlProject( projectFile, null );
311 log.info( "Running soapUI tests in project [" + project.getName() + "]" );
312
313 long startTime = System.nanoTime();
314
315
316 for( int c = 0; c < project.getTestSuiteCount(); c++ )
317 {
318 TestSuite suite = project.getTestSuiteAt( c );
319 for( int i = 0; i < suite.getTestCaseCount(); i++ )
320 {
321 suite.getTestCaseAt( i ).addTestRunListener( this );
322 if (junitReport)
323 suite.getTestCaseAt( i ).addTestRunListener(reportCollector);
324 }
325 }
326
327
328 for( int c = 0; c < project.getTestSuiteCount(); c++ )
329 {
330 if( testSuite == null || project.getTestSuiteAt( c ).getName().equalsIgnoreCase( testSuite ))
331 {
332 runSuite( project.getTestSuiteAt( c ));
333 testSuiteCount++;
334
335
336 if( !runningTests.isEmpty() )
337 log.info( "Waiting for " + runningTests.size() + " tests to finish" );
338
339 while( !runningTests.isEmpty() )
340 {
341 Thread.sleep( 100 );
342 }
343 }
344 }
345
346 long timeTaken = (System.nanoTime()-startTime)/1000000;
347
348 if( printReport )
349 {
350 System.out.println();
351 System.out.println( "SoapUI " + SoapUI.SOAPUI_VERSION + " TestCaseRunner Summary" );
352 System.out.println( "-----------------------------" );
353 System.out.println( "Time Taken: " + timeTaken + "ms" );
354 System.out.println( "Total TestSuites: " + testSuiteCount );
355 System.out.println( "Total TestCases: " + testCaseCount + " (" + failedTests.size() + " failed)");
356 System.out.println( "Total TestSteps: " + testStepCount );
357 System.out.println( "Total Request Assertions: " + testAssertionCount );
358 System.out.println( "Total Failed Assertions: " + assertions.size() );
359 System.out.println( "Total Exported Results: " + exportCount );
360 }
361
362 if (junitReport) {
363 reportCollector.saveReports(outputFolder == null ? "" : outputFolder);
364 }
365
366 if( assertions.size() > 0 || failedTests.size() > 0 )
367 {
368 StringBuffer buf = new StringBuffer();
369
370 for( int c = 0; c < assertions.size(); c++ )
371 {
372 WsdlMessageAssertion assertion = assertions.get( c );
373 WsdlTestRequest testRequest = ((WsdlTestRequest)assertion.getAssertable());
374 failedTests.remove( testRequest.getTestCase() );
375
376 buf.append( assertion.getName() + " in [" + testRequest.getName() + "] failed;\n" );
377 buf.append( Arrays.toString( assertion.getErrors() ) + "\n" );
378
379 WsdlTestStepResult result = assertionResults.get( assertion );
380 StringWriter stringWriter = new StringWriter();
381 PrintWriter writer = new PrintWriter( stringWriter );
382 result.writeTo( writer );
383 buf.append( stringWriter.toString() );
384 }
385
386 while( !failedTests.isEmpty() )
387 {
388 buf.append( "TestCase [" + failedTests.remove( 0 ).getName() + "] failed without assertions\n" );
389 }
390
391 throw new Exception( buf.toString() );
392 }
393 }
394
395 /***
396 * Run tests in the specified TestSuite
397 *
398 * @param suite the TestSuite to run
399 */
400
401 public void runSuite(TestSuite suite)
402 {
403 log.info(( "Running soapUI suite [" + suite.getName() + "], runType = " + suite.getRunType()));
404 long start = System.currentTimeMillis();
405 for( int c = 0; c < suite.getTestCaseCount(); c++ )
406 {
407 String name = suite.getTestCaseAt( c ).getName();
408 if( testCase == null || name.equalsIgnoreCase( testCase ))
409 {
410 runTestCase( suite.getTestCaseAt( c ));
411 }
412 else
413 {
414 log.info( "Skipping testcase [" + name + "], filter is [" + testCase + "]");
415 }
416 }
417 log.info( "soapUI suite [" + suite.getName() + "] finished in " + (System.currentTimeMillis()-start) + "ms" );
418 }
419
420 /***
421 * Runs the specified TestCase
422 *
423 * @param testCase the testcase to run
424 */
425
426 private void runTestCase(TestCase testCase)
427 {
428 runningTests.add( testCase );
429 testCase.run( PropertiesMap.EMPTY_MAP, testCase.getTestSuite().getRunType() == TestSuiteRunType.PARALLEL );
430 }
431
432 /***
433 * Sets the soapUI project file containing the tests to run
434 *
435 * @param projectFile the soapUI project file containing the tests to run
436 */
437
438 public void setProjectFile(String projectFile)
439 {
440 log.info( "setting projectFile to [" + projectFile + "]" );
441 this.projectFile = projectFile;
442 }
443
444 /***
445 * Sets the testcase to run
446 *
447 * @param testCase the testcase to run
448 */
449
450 public void setTestCase(String testCase)
451 {
452 log.info( "setting testCase to [" + testCase + "]" );
453 this.testCase = testCase;
454 }
455
456 /***
457 * Sets the endpoint to use for all test requests
458 *
459 * @param endpoint the endpoint to use for all test requests
460 */
461
462 public void setEndpoint(String endpoint)
463 {
464 log.info( "setting test endpoint to [" + endpoint+ "]" );
465 this.endpoint = endpoint.trim();
466 }
467
468 /***
469 * Sets the TestSuite to run. If not set all TestSuites in the specified project file are run
470 *
471 * @param testSuite the testSuite to run.
472 */
473
474 public void setTestSuite(String testSuite)
475 {
476 log.info( "setting testSuite to [" + testSuite + "]" );
477 this.testSuite = testSuite;
478 }
479
480 public void beforeRun(TestRunner testRunner, TestRunContext runContext)
481 {
482 log.info( "Running soapUI testcase [" + testRunner.getTestCase().getName() + "]");
483 }
484
485 public void beforeStep(TestRunner testRunner, TestRunContext runContext)
486 {
487 TestStep currentStep = runContext.getCurrentStep();
488 log.info( "running step [" + currentStep.getName() + "]" );
489
490 if( currentStep instanceof WsdlTestRequestStep )
491 {
492 WsdlTestRequestStep requestStep = (WsdlTestRequestStep) currentStep;
493 if( endpoint != null && endpoint.length() > 0 )
494 {
495 requestStep.getTestRequest().setEndpoint( endpoint );
496 }
497
498 if( host != null && host.length() > 0 )
499 {
500 try
501 {
502 String ep = replaceHost( requestStep.getTestRequest().getEndpoint(), host );
503 requestStep.getTestRequest().setEndpoint( ep );
504 }
505 catch (Exception e)
506 {
507 log.error( "Failed to set host on endpoint", e );
508 }
509 }
510
511 if( username != null && username.length() > 0 )
512 {
513 requestStep.getTestRequest().setUsername( username );
514 }
515
516 if( password != null && password.length() > 0 )
517 {
518 requestStep.getTestRequest().setPassword( password );
519 }
520
521 if( domain != null && domain.length() > 0 )
522 {
523 requestStep.getTestRequest().setDomain( domain );
524 }
525
526 if( wssPasswordType != null && wssPasswordType.length() > 0 )
527 {
528 requestStep.getTestRequest().setWssPasswordType( wssPasswordType.equals( "Digest" ) ?
529 WsdlTestRequest.PW_TYPE_DIGEST : WsdlTestRequest.PW_TYPE_TEXT );
530 }
531 }
532 }
533
534 public void afterStep(TestRunner testRunner, TestRunContext runContext, TestStepResult result )
535 {
536 TestStep currentStep = runContext.getCurrentStep();
537
538 if( currentStep instanceof WsdlTestRequestStep )
539 {
540 WsdlTestRequestStep requestStep = (WsdlTestRequestStep) currentStep;
541 for( int c = 0; c < requestStep.getAssertionCount(); c++ )
542 {
543 WsdlMessageAssertion assertion = requestStep.getAssertionAt( c );
544 log.info( "Assertion [" + assertion.getName() + "] has status " + assertion.getStatus());
545 if( assertion.getStatus() == AssertionStatus.FAILED )
546 {
547 log.info( "ASSERTION FAILED -> " + assertion.getErrors());
548 assertions.add( assertion );
549 assertionResults.put( assertion, ( WsdlTestStepResult ) result );
550 }
551
552 testAssertionCount++;
553 }
554 }
555
556 String countPropertyName = currentStep.getName() + " run count";
557 Long count = (Long) runContext.getProperty( countPropertyName );
558 if( count == null )
559 {
560 count = new Long( 0 );
561 }
562
563 runContext.setProperty( countPropertyName, new Long( count.longValue()+1 ) );
564
565 if( result.getStatus() == TestStepStatus.FAILED || exportAll )
566 {
567 try
568 {
569 String fileName = currentStep.getTestCase().getTestSuite().getName() + "-"
570 + currentStep.getTestCase().getName() + "-" + currentStep.getName() + "-" +
571 count.longValue() + "-" + result.getStatus() + ".txt";;
572
573 if( outputFolder != null )
574 {
575
576 File folder = new File( outputFolder );
577 if( !folder.exists())
578 folder.mkdirs();
579
580 fileName = outputFolder + File.separator + fileName;
581 }
582
583 if( result.getStatus() == TestStepStatus.FAILED )
584 log.error( currentStep.getName() + " failed, exporting to [" + fileName + "]" );
585
586 PrintWriter writer = new PrintWriter(fileName);
587 result.writeTo(writer);
588 writer.close();
589
590 exportCount++;
591 }
592 catch (Exception e)
593 {
594 log.error( "Error saving failed result: " + e, e );
595 }
596 }
597
598 testStepCount++;
599 }
600
601 public void afterRun(TestRunner testRunner, TestRunContext runContext)
602 {
603 log.info( "Finished running soapUI testcase [" + testRunner.getTestCase().getName() + "], time taken: " +
604 testRunner.getTimeTaken() + "ms, status: " + testRunner.getStatus() );
605
606 if( testRunner.getStatus() == Status.FAILED )
607 {
608 failedTests.add( testRunner.getTestCase() );
609 }
610
611 runningTests.remove( testRunner.getTestCase() );
612
613 testCaseCount++;
614 }
615
616 /***
617 * Replaces the host part of the specified endpoint with the specified host
618 *
619 * @param endpoint the endpoint to modify
620 * @param host the host to set
621 * @return the modified endpoint
622 */
623
624 public static String replaceHost(String endpoint, String host)
625 {
626 int ix1 = endpoint.indexOf( "://" );
627 if( ix1 < 0 )
628 return endpoint;
629
630 int ix2 = endpoint.indexOf( ":", ix1+3 );
631 if( ix2 == -1 || host.indexOf( ":") > 0 )
632 {
633 ix2 = endpoint.indexOf( "/", ix1+3 );
634 if( ix2 == ix1+3 )
635 ix2 = -1;
636 }
637
638 return endpoint.substring( 0, ix1 )+ "://" + host + (ix2 == -1 ? "" : endpoint.substring(ix2));
639 }
640 }