View Javadoc

1   /*
2    *  soapUI, copyright (C) 2004-2007 eviware.com 
3    *
4    *  soapUI is free software; you can redistribute it and/or modify it under the 
5    *  terms of version 2.1 of the GNU Lesser General Public License as published by 
6    *  the Free Software Foundation.
7    *
8    *  soapUI is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without 
9    *  even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 
10   *  See the GNU Lesser General Public License for more details at gnu.org.
11   */
12  
13  package com.eviware.soapui.tools;
14  
15  import java.io.File;
16  import java.io.FileOutputStream;
17  import java.io.PrintWriter;
18  import java.io.StringWriter;
19  import java.util.ArrayList;
20  import java.util.Arrays;
21  import java.util.HashMap;
22  import java.util.List;
23  import java.util.Map;
24  
25  import org.apache.commons.cli.CommandLine;
26  
27  import com.eviware.soapui.SoapUI;
28  import com.eviware.soapui.impl.wsdl.WsdlProject;
29  import com.eviware.soapui.impl.wsdl.WsdlTestSuite;
30  import com.eviware.soapui.impl.wsdl.testcase.WsdlTestCase;
31  import com.eviware.soapui.impl.wsdl.teststeps.WsdlMessageAssertion;
32  import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestRequest;
33  import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestRequestStep;
34  import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestRequestStepResult;
35  import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestStepResult;
36  import com.eviware.soapui.model.iface.Attachment;
37  import com.eviware.soapui.model.propertyexpansion.DefaultPropertyExpansionContext;
38  import com.eviware.soapui.model.propertyexpansion.PropertyExpansionContext;
39  import com.eviware.soapui.model.testsuite.AssertionError;
40  import com.eviware.soapui.model.testsuite.TestCase;
41  import com.eviware.soapui.model.testsuite.TestRunContext;
42  import com.eviware.soapui.model.testsuite.TestRunListener;
43  import com.eviware.soapui.model.testsuite.TestRunner;
44  import com.eviware.soapui.model.testsuite.TestStep;
45  import com.eviware.soapui.model.testsuite.TestStepResult;
46  import com.eviware.soapui.model.testsuite.TestSuite;
47  import com.eviware.soapui.model.testsuite.Assertable.AssertionStatus;
48  import com.eviware.soapui.model.testsuite.TestRunner.Status;
49  import com.eviware.soapui.model.testsuite.TestStepResult.TestStepStatus;
50  import com.eviware.soapui.model.testsuite.TestSuite.TestSuiteRunType;
51  import com.eviware.soapui.report.JUnitReportCollector;
52  import com.eviware.soapui.support.Tools;
53  import com.eviware.soapui.support.types.StringToObjectMap;
54  
55  /***
56   * Standalone test-runner used from maven-plugin, can also be used from command-line (see xdocs) or
57   * directly from other classes.
58   * <p>
59   * For standalone usage, set the project file (with setProjectFile) and other desired properties before
60   * calling run</p> 
61   * 
62   * @author Ole.Matzura
63   */
64  
65  public class SoapUITestCaseRunner extends AbstractSoapUIRunner implements TestRunListener
66  {
67  	public static final String TITLE = "soapUI " + SoapUI.SOAPUI_VERSION + " TestCase Runner";
68  	
69  	private String testSuite;
70  	private String testCase;
71  	private List<WsdlMessageAssertion> assertions = new ArrayList<WsdlMessageAssertion>();
72  	private Map<WsdlMessageAssertion,WsdlTestStepResult> assertionResults = new HashMap<WsdlMessageAssertion,WsdlTestStepResult>();
73  	private List<TestCase> runningTests = new ArrayList<TestCase>();
74  	private List<TestCase> failedTests = new ArrayList<TestCase>();
75  	private String endpoint;
76  	private String domain;
77  	private String password;
78  	private String username;
79  	private String host;
80  	
81  	private int testSuiteCount;
82  	private int testCaseCount;
83  	private int testStepCount;
84  	private int testAssertionCount;
85  
86  	private boolean printReport;
87  	private String outputFolder;
88  	private boolean exportAll;
89  	private boolean junitReport;
90  	private int exportCount;
91  	private JUnitReportCollector reportCollector;
92  	private String wssPasswordType;
93  	private WsdlProject project;
94  	
95  	/***
96  	 * Runs the tests in the specified soapUI project file, see soapUI xdocs for details.
97  	 * 
98  	 * @param args
99  	 * @throws Exception
100 	 */
101 
102 	@SuppressWarnings("static-access")
103 	public static void main( String [] args) throws Exception
104 	{
105 		new SoapUITestCaseRunner().runFromCommandLine( args );
106 	}
107 	
108 	protected boolean processCommandLine( CommandLine cmd )
109 	{
110 		if( cmd.hasOption( "e"))
111 			setEndpoint( cmd.getOptionValue( "e" ) );
112 		
113 		if( cmd.hasOption( "s"))
114 			setTestSuite( cmd.getOptionValue( "s") );
115 
116 		if( cmd.hasOption( "c"))
117 			setTestCase( cmd.getOptionValue( "c") );
118 
119 		if( cmd.hasOption( "u"))
120 			setUsername( cmd.getOptionValue( "u") );
121 
122 		if( cmd.hasOption( "p"))
123 			setPassword( cmd.getOptionValue( "p") );
124 		
125 		if( cmd.hasOption( "w"))
126 			setWssPasswordType( cmd.getOptionValue( "w") );
127 
128 		if( cmd.hasOption( "d"))
129 			setDomain( cmd.getOptionValue( "d") );
130 
131 		if( cmd.hasOption( "h"))
132 			setHost( cmd.getOptionValue( "h") );
133 		
134 		if( cmd.hasOption( "f"))
135 			setOutputFolder( cmd.getOptionValue( "f") );
136 		
137 		if( cmd.hasOption( "t"))
138 			setSettingsFile( cmd.getOptionValue( "t" ));
139 		
140 		setEnableUI( cmd.hasOption( "i") );
141 		setPrintReport( cmd.hasOption( "r" ) );
142 		setExportAll( cmd.hasOption( "a" ) );
143 		setJUnitReport( cmd.hasOption( "j" ) );
144 		
145 		return true;
146 	}
147 
148 	protected SoapUIOptions initCommandLineOptions()
149 	{
150 		SoapUIOptions options = new SoapUIOptions( "testrunner" );
151 		options.addOption( "e", true, "Sets the endpoint" );
152 		options.addOption( "s", true, "Sets the testsuite" );
153 		options.addOption( "c", true, "Sets the testcase" );
154 		options.addOption( "u", true, "Sets the username" );
155 		options.addOption( "p", true, "Sets the password" );
156 		options.addOption( "w", true, "Sets the WSS password type, either 'Text' or 'Digest'" );
157 		options.addOption( "i", false, "Enables Swing UI for scripts" );
158 		options.addOption( "d", true, "Sets the domain" );
159 		options.addOption( "h", true, "Sets the host" );
160 		options.addOption( "r", false, "Prints a small summary report" );
161 		options.addOption( "f", true, "Sets the output folder to export results to" );
162 		options.addOption( "j", false, "Sets the output to include JUnit XML reports" );
163 		options.addOption( "a", false, "Turns on exporting of all results" );
164 		options.addOption( "t", true, "Sets the soapui-settings.xml file to use" );
165 		return options;
166 	}
167 
168 	/***
169 	 * Add console appender to groovy log
170 	 */
171 	
172 	
173 	public void setExportAll(boolean exportAll)
174 	{
175 		this.exportAll = exportAll;
176 	}
177 
178 	public void setJUnitReport(boolean junitReport)
179 	{
180 		this.junitReport = junitReport;
181 		if (junitReport)
182 			reportCollector = new JUnitReportCollector();
183 	}
184 	
185 	public void setOutputFolder(String outputFolder)
186 	{
187 		this.outputFolder = outputFolder;
188 	}
189 	
190 	public SoapUITestCaseRunner()
191 	{
192 		super( SoapUITestCaseRunner.TITLE  );
193 	}
194 	
195 	public SoapUITestCaseRunner( String title )
196 	{
197 		super( title  );
198 	}
199 	
200 	/***
201 	 * Controls if a short test summary should be printed after the test runs
202 	 * 
203 	 * @param printReport a flag controlling if a summary should be printed
204 	 */
205 	
206 	public void setPrintReport(boolean printReport)
207 	{
208 		this.printReport = printReport;
209 	}
210 
211 	/***
212 	 * Sets the host to use by all test-requests, the existing endpoint port and path will be used
213 	 * 
214 	 * @param host the host to use by all requests
215 	 */
216 
217 	public void setHost(String host)
218 	{
219 		this.host = host;
220 	}
221 
222 	/***
223 	 * Sets the domain to use for any authentications
224 	 * 
225 	 * @param domain the domain to use for any authentications
226 	 */
227 	
228 	public void setDomain(String domain)
229 	{
230 		this.domain = domain;
231 	}
232 
233 	/***
234 	 * Sets the password to use for any authentications
235 	 * 
236 	 * @param domain the password to use for any authentications
237 	 */
238 	
239 	public void setPassword(String password)
240 	{
241 		this.password = password;
242 	}
243 	
244 	/***
245 	 * Sets the WSS password-type to use for any authentications. Setting this will result
246 	 * in the addition of WS-Security UsernamePassword tokens to any outgoing request containing
247 	 * the specified username and password.
248 	 * 
249 	 * @param wssPasswordType the wss-password type to use, either 'Text' or 'Digest'
250 	 */
251 	
252 	public void setWssPasswordType( String wssPasswordType )
253 	{
254 		this.wssPasswordType = wssPasswordType;
255 	}
256 
257 	/***
258 	 * Sets the username to use for any authentications
259 	 * 
260 	 * @param domain the username to use for any authentications
261 	 */
262 	
263 	public void setUsername(String username)
264 	{
265 		this.username = username;
266 	}
267 	
268 	/***
269 	 * Runs the testcases as configured with setXXX methods
270 	 * 
271 	 * @throws Exception thrown if any tests fail
272 	 */
273 
274 	public boolean runRunner() throws Exception
275 	{
276 		initGroovyLog();
277 		
278 		if( outputFolder != null )
279 		{
280 			//	ensure folder exists
281 			File folder = new File( outputFolder );
282 			if( !folder.exists())
283 				folder.mkdirs();
284 		}
285 		
286 		assertions.clear();
287 		
288 		String projectFile = getProjectFile();
289 		
290 		project = new WsdlProject( projectFile );
291 		if( project.isDisabled() )
292 			throw new Exception( "Failed to load soapUI project file [" + projectFile + "]" );
293 		
294 		log.info( "Running soapUI tests in project [" + project.getName() + "]" );
295 		
296 		long startTime = System.nanoTime();
297 		
298 		// start by listening to all testcases.. (since one testcase can call another)
299 		for( int c = 0; c < project.getTestSuiteCount(); c++ )
300 		{
301 			TestSuite suite = project.getTestSuiteAt( c );
302 			for( int i = 0; i < suite.getTestCaseCount(); i++ )
303 			{
304 				suite.getTestCaseAt( i ).addTestRunListener( this );
305 				if (junitReport)
306 					suite.getTestCaseAt( i ).addTestRunListener(reportCollector);
307 			}
308 		}
309 		
310 		// now run tests..
311 		for( int c = 0; c < project.getTestSuiteCount(); c++ )
312 		{
313 			WsdlTestSuite ts = project.getTestSuiteAt( c );
314 			if( !ts.isDisabled() && (testSuite == null || ts.getName().equalsIgnoreCase( testSuite )))
315 			{
316 				runSuite( ts);
317 				testSuiteCount++;
318 				
319 				//	wait for tests to finish if running in parallell mode
320 				if( !runningTests.isEmpty() )
321 					log.info( "Waiting for " + runningTests.size() + " tests to finish" );
322 				
323 				while( !runningTests.isEmpty() )
324 				{
325 					Thread.sleep( 100 );
326 				}
327 			}
328 		}
329 		
330 		long timeTaken = (System.nanoTime()-startTime)/1000000;
331 		
332 		if( printReport )
333 		{
334 			printReport( timeTaken );
335 		}
336 		
337 		if (junitReport) {
338 			exportJUnitReports( reportCollector, outputFolder );
339 		}
340 		
341 		if( assertions.size() > 0 || failedTests.size() > 0 )
342 		{
343 			throwFailureException();
344 		}
345 		
346 		return true;
347 	}
348 
349 	protected void throwFailureException() throws Exception
350 	{
351 		StringBuffer buf = new StringBuffer();
352 		
353 		for( int c = 0; c < assertions.size(); c++ )
354 		{
355 			WsdlMessageAssertion assertion = assertions.get( c );
356 			WsdlTestRequest testRequest = ((WsdlTestRequest)assertion.getAssertable());
357 			failedTests.remove( testRequest.getTestCase() );
358 			
359 			buf.append( assertion.getName() + " in [" + testRequest.getName() + "] failed;\n" );
360 			buf.append( Arrays.toString( assertion.getErrors() ) + "\n" );
361 			
362 			WsdlTestStepResult result = assertionResults.get( assertion  );
363 			StringWriter stringWriter = new StringWriter();
364 			PrintWriter writer = new PrintWriter( stringWriter );
365 			result.writeTo( writer );
366 			buf.append( stringWriter.toString() );
367 		}
368 
369 		while( !failedTests.isEmpty() )
370 		{
371 			buf.append( "TestCase [" + failedTests.remove( 0 ).getName() + "] failed without assertions\n" );
372 		}
373 		
374 		throw new Exception( buf.toString() );
375 	}
376 
377 	public void exportJUnitReports( JUnitReportCollector collector, String folder ) throws Exception
378 	{
379 		collector.saveReports(folder == null ? "" : folder);
380 	}
381 
382 	public void printReport( long timeTaken )
383 	{
384 		System.out.println();
385 		System.out.println( "SoapUI " + SoapUI.SOAPUI_VERSION + " TestCaseRunner Summary" );
386 		System.out.println( "-----------------------------" );
387 		System.out.println( "Time Taken: " + timeTaken + "ms" );
388 		System.out.println( "Total TestSuites: " + testSuiteCount );
389 		System.out.println( "Total TestCases: " + testCaseCount + " (" + failedTests.size() + " failed)");
390 		System.out.println( "Total TestSteps: " + testStepCount );
391 		System.out.println( "Total Request Assertions: " + testAssertionCount );
392 		System.out.println( "Total Failed Assertions: " + assertions.size() );
393 		System.out.println( "Total Exported Results: " + exportCount );
394 	}
395 
396 	/***
397 	 * Run tests in the specified TestSuite
398 	 *
399 	 * @param suite the TestSuite to run
400 	 */
401 	
402 	public void runSuite(WsdlTestSuite suite)
403 	{
404 		log.info(( "Running soapUI suite [" + suite.getName() + "], runType = " + suite.getRunType()));
405 		PropertyExpansionContext context = new DefaultPropertyExpansionContext( suite );
406 		long start = System.currentTimeMillis();
407 		
408 		try
409 		{
410 			suite.runSetupScript( context );
411 			for( int c = 0; c < suite.getTestCaseCount(); c++ )
412 			{
413 				WsdlTestCase tc = suite.getTestCaseAt( c );
414 				
415 				String name = tc.getName();
416 				if( testCase == null || name.equalsIgnoreCase( testCase ))
417 				{
418 					if( !tc.isDisabled() )
419 					{
420 						runTestCase( tc);
421 					}
422 					else
423 					{
424 						log.info( "Skipping disabled testcase [" + name + "]");
425 					}
426 				}
427 				else
428 				{
429 					log.info( "Skipping testcase [" + name + "], filter is [" + testCase + "]");
430 				}
431 			}
432 		}
433 		catch( Exception e )
434 		{
435 			e.printStackTrace();
436 		}
437 		finally
438 		{
439 			try
440 			{
441 				suite.runTearDownScript( context );
442 			}
443 			catch( Exception e )
444 			{
445 				e.printStackTrace();
446 			}
447 		}
448 		
449 		log.info( "soapUI suite [" + suite.getName() + "] finished in " + (System.currentTimeMillis()-start) + "ms" );
450 	}
451 
452 	/***
453 	 * Runs the specified TestCase
454 	 * 
455 	 * @param testCase the testcase to run
456 	 */
457 	
458 	private void runTestCase(TestCase testCase)
459 	{
460 		runningTests.add( testCase );
461 		testCase.run( new StringToObjectMap(),	testCase.getTestSuite().getRunType() == TestSuiteRunType.PARALLEL );
462 	}
463 
464 	/***
465 	 * Sets the testcase to run
466 	 * 
467 	 * @param testCase the testcase to run
468 	 */
469 	
470 	public void setTestCase(String testCase)
471 	{
472       this.testCase = testCase;
473 	}
474 	
475 	/***
476 	 * Sets the endpoint to use for all test requests
477 	 * 
478 	 * @param endpoint the endpoint to use for all test requests
479 	 */
480 	
481 	public void setEndpoint(String endpoint)
482 	{
483 		this.endpoint = endpoint.trim();
484 	}
485 	
486 	/***
487 	 * Sets the TestSuite to run. If not set all TestSuites in the specified project file are run
488 	 * 
489 	 * @param testSuite the testSuite to run.
490 	 */
491 
492 	public void setTestSuite(String testSuite)
493 	{
494 		this.testSuite = testSuite;
495 	}
496 	
497 	public void beforeRun(TestRunner testRunner, TestRunContext runContext)
498 	{
499 		log.info( "Running soapUI testcase [" + testRunner.getTestCase().getName() + "]");
500 	}
501 
502 	public void beforeStep(TestRunner testRunner, TestRunContext runContext)
503 	{
504 		TestStep currentStep = runContext.getCurrentStep();
505 		log.info( "running step [" + currentStep.getName() + "]" );
506 		
507 		if( currentStep instanceof WsdlTestRequestStep )
508 		{
509 			WsdlTestRequestStep requestStep = (WsdlTestRequestStep) currentStep;
510 			if( endpoint != null && endpoint.length() > 0 )
511 			{
512 				requestStep.getTestRequest().setEndpoint( endpoint );
513 			}
514 			
515 			if( host != null && host.length() > 0 )
516 			{
517 				try
518 				{
519 					String ep = Tools.replaceHost( requestStep.getTestRequest().getEndpoint(), host );
520 					requestStep.getTestRequest().setEndpoint( ep );
521 				}
522 				catch (Exception e)
523 				{
524 					log.error( "Failed to set host on endpoint", e );
525 				}				
526 			}
527 
528 			if( username != null && username.length() > 0 )
529 			{
530 				requestStep.getTestRequest().setUsername( username );
531 			}
532 			
533 			if( password != null && password.length() > 0 )
534 			{
535 				requestStep.getTestRequest().setPassword( password );
536 			}
537 			
538 			if( domain != null && domain.length() > 0 )
539 			{
540 				requestStep.getTestRequest().setDomain( domain );
541 			}
542 			
543 			if( wssPasswordType != null && wssPasswordType.length() > 0 )
544 			{
545 				requestStep.getTestRequest().setWssPasswordType( wssPasswordType.equals( "Digest" ) ? 
546 							WsdlTestRequest.PW_TYPE_DIGEST : WsdlTestRequest.PW_TYPE_TEXT );
547 			}
548 		}
549 	}
550 
551 	public void afterStep(TestRunner testRunner, TestRunContext runContext, TestStepResult result )
552 	{
553 		TestStep currentStep = runContext.getCurrentStep();
554 		
555 		if( currentStep instanceof WsdlTestRequestStep )
556 		{
557 			WsdlTestRequestStep requestStep = (WsdlTestRequestStep) currentStep;
558 			for( int c = 0; c < requestStep.getAssertionCount(); c++ )
559 			{
560 				WsdlMessageAssertion assertion = requestStep.getAssertionAt( c );
561 				log.info( "Assertion [" + assertion.getName() + "] has status " + assertion.getStatus());
562 				if( assertion.getStatus() == AssertionStatus.FAILED )
563 				{
564 					for( AssertionError error : assertion.getErrors())
565 						log.error( "ASSERTION FAILED -> " + error.getMessage());
566 					
567 					assertions.add( assertion );
568 					assertionResults.put( assertion, ( WsdlTestStepResult ) result );
569 				}
570 				
571 				testAssertionCount++;
572 			}
573 		}	
574 		
575 		String countPropertyName = currentStep.getName() + " run count";
576 		Long count = (Long) runContext.getProperty( countPropertyName );
577 		if( count == null )
578 		{
579 			count = new Long( 0 );
580 		}
581 
582 		runContext.setProperty( countPropertyName, new Long( count.longValue()+1 ) );
583 		
584 		if( result.getStatus() == TestStepStatus.FAILED || exportAll )
585 		{
586 			try
587 			{
588 				String nameBase = currentStep.getTestCase().getTestSuite().getName() + "-"
589 														  + currentStep.getTestCase().getName() + "-" + currentStep.getName() + "-" +	
590 														  count.longValue() + "-" + result.getStatus();
591 				
592 				String fileName = nameBase + ".txt";
593 				if( outputFolder != null )
594 				{
595 					fileName = outputFolder + File.separator + fileName;
596 				}
597 				
598 				if( result.getStatus() == TestStepStatus.FAILED )
599 					log.error( currentStep.getName() + " failed, exporting to [" + fileName + "]" );
600 				
601 				PrintWriter writer = new PrintWriter(fileName);
602 				result.writeTo(writer);
603 				writer.close();
604 				
605 				// write attachments
606 				if( result instanceof WsdlTestRequestStepResult )
607 				{
608 					Attachment [] attachments = ((WsdlTestRequestStepResult)result).getResponseAttachments();
609 					if( attachments != null && attachments.length > 0 )
610 					{
611 						for( int c = 0; c < attachments.length; c++ )
612 						{
613 							fileName = nameBase + "-attachment-" + (c+1) + ".";
614 							
615 							Attachment attachment = attachments[c];
616 							String contentType = attachment.getContentType();
617 							if( !"application/octet-stream".equals( contentType ) && contentType != null && contentType.indexOf( '/' ) != -1 )
618 							{
619 								fileName += contentType.substring( contentType.lastIndexOf( '/' )+1 );
620 							}
621 							else
622 							{
623 								fileName += "dat";
624 							}
625 							
626 							if( outputFolder != null )
627 							{
628 								fileName = outputFolder + File.separator + fileName;
629 							}
630 							
631 							FileOutputStream outFile = new FileOutputStream( fileName );
632 							Tools.writeAll( outFile, attachment.getInputStream() );
633 							outFile.close();
634 						}
635 					}
636 				}
637 				
638 				exportCount++;
639 			}
640 			catch (Exception e)
641 			{
642 				log.error( "Error saving failed result: " + e, e );
643 			}			
644 		}
645 		
646 		testStepCount++;
647 	}
648 
649 	public void afterRun(TestRunner testRunner, TestRunContext runContext)
650 	{
651 		log.info( "Finished running soapUI testcase [" + testRunner.getTestCase().getName() + "], time taken: " + 
652 				testRunner.getTimeTaken() + "ms, status: " + testRunner.getStatus() );
653 		
654 		if( testRunner.getStatus() == Status.FAILED )
655 		{
656 			failedTests.add( testRunner.getTestCase() );
657 		}
658 		
659 		runningTests.remove( testRunner.getTestCase() );
660 		
661 		testCaseCount++;
662 	}
663 
664 	protected WsdlProject getProject()
665 	{
666 		return project;
667 	}
668 }