1
2
3
4
5
6
7
8
9
10
11
12
13 package com.eviware.soapui.tools;
14
15 import java.io.File;
16 import java.io.FileOutputStream;
17 import java.io.PrintWriter;
18 import java.io.StringWriter;
19 import java.util.ArrayList;
20 import java.util.Arrays;
21 import java.util.HashMap;
22 import java.util.List;
23 import java.util.Map;
24
25 import org.apache.commons.cli.CommandLine;
26
27 import com.eviware.soapui.SoapUI;
28 import com.eviware.soapui.impl.wsdl.WsdlProject;
29 import com.eviware.soapui.impl.wsdl.support.assertions.Assertable.AssertionStatus;
30 import com.eviware.soapui.impl.wsdl.teststeps.WsdlMessageAssertion;
31 import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestRequest;
32 import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestRequestStep;
33 import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestRequestStepResult;
34 import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestStepResult;
35 import com.eviware.soapui.impl.wsdl.teststeps.assertions.AssertionError;
36 import com.eviware.soapui.model.iface.Attachment;
37 import com.eviware.soapui.model.support.PropertiesMap;
38 import com.eviware.soapui.model.testsuite.TestCase;
39 import com.eviware.soapui.model.testsuite.TestRunContext;
40 import com.eviware.soapui.model.testsuite.TestRunListener;
41 import com.eviware.soapui.model.testsuite.TestRunner;
42 import com.eviware.soapui.model.testsuite.TestStep;
43 import com.eviware.soapui.model.testsuite.TestStepResult;
44 import com.eviware.soapui.model.testsuite.TestSuite;
45 import com.eviware.soapui.model.testsuite.TestRunner.Status;
46 import com.eviware.soapui.model.testsuite.TestStepResult.TestStepStatus;
47 import com.eviware.soapui.model.testsuite.TestSuite.TestSuiteRunType;
48 import com.eviware.soapui.report.JUnitReportCollector;
49 import com.eviware.soapui.support.Tools;
50
51 /***
52 * Standalone test-runner used from maven-plugin, can also be used from command-line (see xdocs) or
53 * directly from other classes.
54 * <p>
55 * For standalone usage, set the project file (with setProjectFile) and other desired properties before
56 * calling run</p>
57 *
58 * @author Ole.Matzura
59 */
60
61 public class SoapUITestCaseRunner extends AbstractSoapUIRunner implements TestRunListener
62 {
63 public static final String TITLE = "soapUI " + SoapUI.SOAPUI_VERSION + " TestCase Runner";
64
65 private String testSuite;
66 private String testCase;
67 private List<WsdlMessageAssertion> assertions = new ArrayList<WsdlMessageAssertion>();
68 private Map<WsdlMessageAssertion,WsdlTestStepResult> assertionResults = new HashMap<WsdlMessageAssertion,WsdlTestStepResult>();
69 private List<TestCase> runningTests = new ArrayList<TestCase>();
70 private List<TestCase> failedTests = new ArrayList<TestCase>();
71 private String endpoint;
72 private String domain;
73 private String password;
74 private String username;
75 private String host;
76
77 private int testSuiteCount;
78 private int testCaseCount;
79 private int testStepCount;
80 private int testAssertionCount;
81
82 private boolean printReport;
83 private String outputFolder;
84 private boolean exportAll;
85 private boolean junitReport;
86 private int exportCount;
87 private JUnitReportCollector reportCollector;
88 private String wssPasswordType;
89 private WsdlProject project;
90
91 /***
92 * Runs the tests in the specified soapUI project file, see soapUI xdocs for details.
93 *
94 * @param args
95 * @throws Exception
96 */
97
98 @SuppressWarnings("static-access")
99 public static void main( String [] args) throws Exception
100 {
101 new SoapUITestCaseRunner().runFromCommandLine( args );
102 }
103
104 protected boolean processCommandLine( CommandLine cmd )
105 {
106 if( cmd.hasOption( "e"))
107 setEndpoint( cmd.getOptionValue( "e" ) );
108
109 if( cmd.hasOption( "s"))
110 setTestSuite( cmd.getOptionValue( "s") );
111
112 if( cmd.hasOption( "c"))
113 setTestCase( cmd.getOptionValue( "c") );
114
115 if( cmd.hasOption( "u"))
116 setUsername( cmd.getOptionValue( "u") );
117
118 if( cmd.hasOption( "p"))
119 setPassword( cmd.getOptionValue( "p") );
120
121 if( cmd.hasOption( "w"))
122 setWssPasswordType( cmd.getOptionValue( "w") );
123
124 if( cmd.hasOption( "d"))
125 setDomain( cmd.getOptionValue( "d") );
126
127 if( cmd.hasOption( "h"))
128 setHost( cmd.getOptionValue( "h") );
129
130 if( cmd.hasOption( "f"))
131 setOutputFolder( cmd.getOptionValue( "f") );
132
133 if( cmd.hasOption( "t"))
134 setSettingsFile( cmd.getOptionValue( "t" ));
135
136 setEnableUI( cmd.hasOption( "i") );
137 setPrintReport( cmd.hasOption( "r" ) );
138 setExportAll( cmd.hasOption( "a" ) );
139 setJUnitReport( cmd.hasOption( "j" ) );
140
141 return true;
142 }
143
144 protected SoapUIOptions initCommandLineOptions()
145 {
146 SoapUIOptions options = new SoapUIOptions( "testrunner" );
147 options.addOption( "e", true, "Sets the endpoint" );
148 options.addOption( "s", true, "Sets the testsuite" );
149 options.addOption( "c", true, "Sets the testcase" );
150 options.addOption( "u", true, "Sets the username" );
151 options.addOption( "p", true, "Sets the password" );
152 options.addOption( "w", true, "Sets the WSS password type, either 'Text' or 'Digest'" );
153 options.addOption( "i", false, "Enables Swing UI for scripts" );
154 options.addOption( "d", true, "Sets the domain" );
155 options.addOption( "h", true, "Sets the host" );
156 options.addOption( "r", false, "Prints a small summary report" );
157 options.addOption( "f", true, "Sets the output folder to export results to" );
158 options.addOption( "j", false, "Sets the output to include JUnit XML reports" );
159 options.addOption( "a", false, "Turns on exporting of all results" );
160 options.addOption( "t", true, "Sets the soapui-settings.xml file to use" );
161 return options;
162 }
163
164 /***
165 * Add console appender to groovy log
166 */
167
168
169 public void setExportAll(boolean exportAll)
170 {
171 this.exportAll = exportAll;
172 }
173
174 public void setJUnitReport(boolean junitReport)
175 {
176 this.junitReport = junitReport;
177 if (junitReport)
178 reportCollector = new JUnitReportCollector();
179 }
180
181 public void setOutputFolder(String outputFolder)
182 {
183 this.outputFolder = outputFolder;
184 }
185
186 public SoapUITestCaseRunner()
187 {
188 super( SoapUITestCaseRunner.TITLE );
189 }
190
191 public SoapUITestCaseRunner( String title )
192 {
193 super( title );
194 }
195
196 /***
197 * Controls if a short test summary should be printed after the test runs
198 *
199 * @param printReport a flag controlling if a summary should be printed
200 */
201
202 public void setPrintReport(boolean printReport)
203 {
204 this.printReport = printReport;
205 }
206
207 /***
208 * Sets the host to use by all test-requests, the existing endpoint port and path will be used
209 *
210 * @param host the host to use by all requests
211 */
212
213 public void setHost(String host)
214 {
215 this.host = host;
216 }
217
218 /***
219 * Sets the domain to use for any authentications
220 *
221 * @param domain the domain to use for any authentications
222 */
223
224 public void setDomain(String domain)
225 {
226 this.domain = domain;
227 }
228
229 /***
230 * Sets the password to use for any authentications
231 *
232 * @param domain the password to use for any authentications
233 */
234
235 public void setPassword(String password)
236 {
237 this.password = password;
238 }
239
240 /***
241 * Sets the WSS password-type to use for any authentications. Setting this will result
242 * in the addition of WS-Security UsernamePassword tokens to any outgoing request containing
243 * the specified username and password.
244 *
245 * @param wssPasswordType the wss-password type to use, either 'Text' or 'Digest'
246 */
247
248 public void setWssPasswordType( String wssPasswordType )
249 {
250 this.wssPasswordType = wssPasswordType;
251 }
252
253 /***
254 * Sets the username to use for any authentications
255 *
256 * @param domain the username to use for any authentications
257 */
258
259 public void setUsername(String username)
260 {
261 this.username = username;
262 }
263
264 /***
265 * Runs the testcases as configured with setXXX methods
266 *
267 * @throws Exception thrown if any tests fail
268 */
269
270 public void runRunner() throws Exception
271 {
272 initGroovyLog();
273
274 if( outputFolder != null )
275 {
276
277 File folder = new File( outputFolder );
278 if( !folder.exists())
279 folder.mkdirs();
280 }
281
282 assertions.clear();
283
284 String projectFile = getProjectFile();
285
286 project = new WsdlProject( projectFile );
287 if( project.isDisabled() )
288 throw new Exception( "Failed to load soapUI project file [" + projectFile + "]" );
289
290 log.info( "Running soapUI tests in project [" + project.getName() + "]" );
291
292 long startTime = System.nanoTime();
293
294
295 for( int c = 0; c < project.getTestSuiteCount(); c++ )
296 {
297 TestSuite suite = project.getTestSuiteAt( c );
298 for( int i = 0; i < suite.getTestCaseCount(); i++ )
299 {
300 suite.getTestCaseAt( i ).addTestRunListener( this );
301 if (junitReport)
302 suite.getTestCaseAt( i ).addTestRunListener(reportCollector);
303 }
304 }
305
306
307 for( int c = 0; c < project.getTestSuiteCount(); c++ )
308 {
309 if( testSuite == null || project.getTestSuiteAt( c ).getName().equalsIgnoreCase( testSuite ))
310 {
311 runSuite( project.getTestSuiteAt( c ));
312 testSuiteCount++;
313
314
315 if( !runningTests.isEmpty() )
316 log.info( "Waiting for " + runningTests.size() + " tests to finish" );
317
318 while( !runningTests.isEmpty() )
319 {
320 Thread.sleep( 100 );
321 }
322 }
323 }
324
325 long timeTaken = (System.nanoTime()-startTime)/1000000;
326
327 if( printReport )
328 {
329 printReport( timeTaken );
330 }
331
332 if (junitReport) {
333 exportJUnitReports( reportCollector, outputFolder );
334 }
335
336 if( assertions.size() > 0 || failedTests.size() > 0 )
337 {
338 throwFailureException();
339 }
340 }
341
342 protected void throwFailureException() throws Exception
343 {
344 StringBuffer buf = new StringBuffer();
345
346 for( int c = 0; c < assertions.size(); c++ )
347 {
348 WsdlMessageAssertion assertion = assertions.get( c );
349 WsdlTestRequest testRequest = ((WsdlTestRequest)assertion.getAssertable());
350 failedTests.remove( testRequest.getTestCase() );
351
352 buf.append( assertion.getName() + " in [" + testRequest.getName() + "] failed;\n" );
353 buf.append( Arrays.toString( assertion.getErrors() ) + "\n" );
354
355 WsdlTestStepResult result = assertionResults.get( assertion );
356 StringWriter stringWriter = new StringWriter();
357 PrintWriter writer = new PrintWriter( stringWriter );
358 result.writeTo( writer );
359 buf.append( stringWriter.toString() );
360 }
361
362 while( !failedTests.isEmpty() )
363 {
364 buf.append( "TestCase [" + failedTests.remove( 0 ).getName() + "] failed without assertions\n" );
365 }
366
367 throw new Exception( buf.toString() );
368 }
369
370 public void exportJUnitReports( JUnitReportCollector collector, String folder ) throws Exception
371 {
372 collector.saveReports(folder == null ? "" : folder);
373 }
374
375 public void printReport( long timeTaken )
376 {
377 System.out.println();
378 System.out.println( "SoapUI " + SoapUI.SOAPUI_VERSION + " TestCaseRunner Summary" );
379 System.out.println( "-----------------------------" );
380 System.out.println( "Time Taken: " + timeTaken + "ms" );
381 System.out.println( "Total TestSuites: " + testSuiteCount );
382 System.out.println( "Total TestCases: " + testCaseCount + " (" + failedTests.size() + " failed)");
383 System.out.println( "Total TestSteps: " + testStepCount );
384 System.out.println( "Total Request Assertions: " + testAssertionCount );
385 System.out.println( "Total Failed Assertions: " + assertions.size() );
386 System.out.println( "Total Exported Results: " + exportCount );
387 }
388
389 /***
390 * Run tests in the specified TestSuite
391 *
392 * @param suite the TestSuite to run
393 */
394
395 public void runSuite(TestSuite suite)
396 {
397 log.info(( "Running soapUI suite [" + suite.getName() + "], runType = " + suite.getRunType()));
398 long start = System.currentTimeMillis();
399 for( int c = 0; c < suite.getTestCaseCount(); c++ )
400 {
401 String name = suite.getTestCaseAt( c ).getName();
402 if( testCase == null || name.equalsIgnoreCase( testCase ))
403 {
404 runTestCase( suite.getTestCaseAt( c ));
405 }
406 else
407 {
408 log.info( "Skipping testcase [" + name + "], filter is [" + testCase + "]");
409 }
410 }
411 log.info( "soapUI suite [" + suite.getName() + "] finished in " + (System.currentTimeMillis()-start) + "ms" );
412 }
413
414 /***
415 * Runs the specified TestCase
416 *
417 * @param testCase the testcase to run
418 */
419
420 private void runTestCase(TestCase testCase)
421 {
422 runningTests.add( testCase );
423 testCase.run( PropertiesMap.EMPTY_MAP, testCase.getTestSuite().getRunType() == TestSuiteRunType.PARALLEL );
424 }
425
426 /***
427 * Sets the testcase to run
428 *
429 * @param testCase the testcase to run
430 */
431
432 public void setTestCase(String testCase)
433 {
434 this.testCase = testCase;
435 }
436
437 /***
438 * Sets the endpoint to use for all test requests
439 *
440 * @param endpoint the endpoint to use for all test requests
441 */
442
443 public void setEndpoint(String endpoint)
444 {
445 this.endpoint = endpoint.trim();
446 }
447
448 /***
449 * Sets the TestSuite to run. If not set all TestSuites in the specified project file are run
450 *
451 * @param testSuite the testSuite to run.
452 */
453
454 public void setTestSuite(String testSuite)
455 {
456 this.testSuite = testSuite;
457 }
458
459 public void beforeRun(TestRunner testRunner, TestRunContext runContext)
460 {
461 log.info( "Running soapUI testcase [" + testRunner.getTestCase().getName() + "]");
462 }
463
464 public void beforeStep(TestRunner testRunner, TestRunContext runContext)
465 {
466 TestStep currentStep = runContext.getCurrentStep();
467 log.info( "running step [" + currentStep.getName() + "]" );
468
469 if( currentStep instanceof WsdlTestRequestStep )
470 {
471 WsdlTestRequestStep requestStep = (WsdlTestRequestStep) currentStep;
472 if( endpoint != null && endpoint.length() > 0 )
473 {
474 requestStep.getTestRequest().setEndpoint( endpoint );
475 }
476
477 if( host != null && host.length() > 0 )
478 {
479 try
480 {
481 String ep = Tools.replaceHost( requestStep.getTestRequest().getEndpoint(), host );
482 requestStep.getTestRequest().setEndpoint( ep );
483 }
484 catch (Exception e)
485 {
486 log.error( "Failed to set host on endpoint", e );
487 }
488 }
489
490 if( username != null && username.length() > 0 )
491 {
492 requestStep.getTestRequest().setUsername( username );
493 }
494
495 if( password != null && password.length() > 0 )
496 {
497 requestStep.getTestRequest().setPassword( password );
498 }
499
500 if( domain != null && domain.length() > 0 )
501 {
502 requestStep.getTestRequest().setDomain( domain );
503 }
504
505 if( wssPasswordType != null && wssPasswordType.length() > 0 )
506 {
507 requestStep.getTestRequest().setWssPasswordType( wssPasswordType.equals( "Digest" ) ?
508 WsdlTestRequest.PW_TYPE_DIGEST : WsdlTestRequest.PW_TYPE_TEXT );
509 }
510 }
511 }
512
513 public void afterStep(TestRunner testRunner, TestRunContext runContext, TestStepResult result )
514 {
515 TestStep currentStep = runContext.getCurrentStep();
516
517 if( currentStep instanceof WsdlTestRequestStep )
518 {
519 WsdlTestRequestStep requestStep = (WsdlTestRequestStep) currentStep;
520 for( int c = 0; c < requestStep.getAssertionCount(); c++ )
521 {
522 WsdlMessageAssertion assertion = requestStep.getAssertionAt( c );
523 log.info( "Assertion [" + assertion.getName() + "] has status " + assertion.getStatus());
524 if( assertion.getStatus() == AssertionStatus.FAILED )
525 {
526 for( AssertionError error : assertion.getErrors())
527 log.error( "ASSERTION FAILED -> " + error.getMessage());
528
529 assertions.add( assertion );
530 assertionResults.put( assertion, ( WsdlTestStepResult ) result );
531 }
532
533 testAssertionCount++;
534 }
535 }
536
537 String countPropertyName = currentStep.getName() + " run count";
538 Long count = (Long) runContext.getProperty( countPropertyName );
539 if( count == null )
540 {
541 count = new Long( 0 );
542 }
543
544 runContext.setProperty( countPropertyName, new Long( count.longValue()+1 ) );
545
546 if( result.getStatus() == TestStepStatus.FAILED || exportAll )
547 {
548 try
549 {
550 String nameBase = currentStep.getTestCase().getTestSuite().getName() + "-"
551 + currentStep.getTestCase().getName() + "-" + currentStep.getName() + "-" +
552 count.longValue() + "-" + result.getStatus();
553
554 String fileName = nameBase + ".txt";
555 if( outputFolder != null )
556 {
557 fileName = outputFolder + File.separator + fileName;
558 }
559
560 if( result.getStatus() == TestStepStatus.FAILED )
561 log.error( currentStep.getName() + " failed, exporting to [" + fileName + "]" );
562
563 PrintWriter writer = new PrintWriter(fileName);
564 result.writeTo(writer);
565 writer.close();
566
567
568 if( result instanceof WsdlTestRequestStepResult )
569 {
570 Attachment [] attachments = ((WsdlTestRequestStepResult)result).getResponseAttachments();
571 if( attachments != null && attachments.length > 0 )
572 {
573 for( int c = 0; c < attachments.length; c++ )
574 {
575 fileName = nameBase + "-attachment-" + (c+1) + ".";
576
577 Attachment attachment = attachments[c];
578 String contentType = attachment.getContentType();
579 if( !"application/octet-stream".equals( contentType ) && contentType != null && contentType.indexOf( '/' ) != -1 )
580 {
581 fileName += contentType.substring( contentType.lastIndexOf( '/' )+1 );
582 }
583 else
584 {
585 fileName += "dat";
586 }
587
588 if( outputFolder != null )
589 {
590 fileName = outputFolder + File.separator + fileName;
591 }
592
593 FileOutputStream outFile = new FileOutputStream( fileName );
594 Tools.writeAll( outFile, attachment.getInputStream() );
595 outFile.close();
596 }
597 }
598 }
599
600 exportCount++;
601 }
602 catch (Exception e)
603 {
604 log.error( "Error saving failed result: " + e, e );
605 }
606 }
607
608 testStepCount++;
609 }
610
611 public void afterRun(TestRunner testRunner, TestRunContext runContext)
612 {
613 log.info( "Finished running soapUI testcase [" + testRunner.getTestCase().getName() + "], time taken: " +
614 testRunner.getTimeTaken() + "ms, status: " + testRunner.getStatus() );
615
616 if( testRunner.getStatus() == Status.FAILED )
617 {
618 failedTests.add( testRunner.getTestCase() );
619 }
620
621 runningTests.remove( testRunner.getTestCase() );
622
623 testCaseCount++;
624 }
625
626 protected WsdlProject getProject()
627 {
628 return project;
629 }
630 }