View Javadoc

1   /*
2    *  soapUI, copyright (C) 2004-2007 eviware.com 
3    *
4    *  soapUI is free software; you can redistribute it and/or modify it under the 
5    *  terms of version 2.1 of the GNU Lesser General Public License as published by 
6    *  the Free Software Foundation.
7    *
8    *  soapUI is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without 
9    *  even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 
10   *  See the GNU Lesser General Public License for more details at gnu.org.
11   */
12  
13  package com.eviware.soapui.tools;
14  
15  import java.io.File;
16  import java.io.FileOutputStream;
17  import java.io.PrintWriter;
18  import java.io.StringWriter;
19  import java.util.ArrayList;
20  import java.util.Arrays;
21  import java.util.HashMap;
22  import java.util.List;
23  import java.util.Map;
24  
25  import org.apache.commons.cli.CommandLine;
26  
27  import com.eviware.soapui.SoapUI;
28  import com.eviware.soapui.impl.wsdl.WsdlProject;
29  import com.eviware.soapui.impl.wsdl.WsdlTestSuite;
30  import com.eviware.soapui.impl.wsdl.testcase.WsdlTestCase;
31  import com.eviware.soapui.impl.wsdl.teststeps.WsdlMessageAssertion;
32  import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestRequest;
33  import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestRequestStep;
34  import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestRequestStepResult;
35  import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestStep;
36  import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestStepResult;
37  import com.eviware.soapui.model.iface.Attachment;
38  import com.eviware.soapui.model.propertyexpansion.DefaultPropertyExpansionContext;
39  import com.eviware.soapui.model.propertyexpansion.PropertyExpansionContext;
40  import com.eviware.soapui.model.testsuite.Assertable;
41  import com.eviware.soapui.model.testsuite.AssertionError;
42  import com.eviware.soapui.model.testsuite.TestCase;
43  import com.eviware.soapui.model.testsuite.TestRunContext;
44  import com.eviware.soapui.model.testsuite.TestRunListener;
45  import com.eviware.soapui.model.testsuite.TestRunner;
46  import com.eviware.soapui.model.testsuite.TestStep;
47  import com.eviware.soapui.model.testsuite.TestStepResult;
48  import com.eviware.soapui.model.testsuite.TestSuite;
49  import com.eviware.soapui.model.testsuite.Assertable.AssertionStatus;
50  import com.eviware.soapui.model.testsuite.TestRunner.Status;
51  import com.eviware.soapui.model.testsuite.TestStepResult.TestStepStatus;
52  import com.eviware.soapui.model.testsuite.TestSuite.TestSuiteRunType;
53  import com.eviware.soapui.report.JUnitReportCollector;
54  import com.eviware.soapui.support.Tools;
55  import com.eviware.soapui.support.types.StringToObjectMap;
56  
57  /***
58   * Standalone test-runner used from maven-plugin, can also be used from command-line (see xdocs) or
59   * directly from other classes.
60   * <p>
61   * For standalone usage, set the project file (with setProjectFile) and other desired properties before
62   * calling run</p> 
63   * 
64   * @author Ole.Matzura
65   */
66  
67  public class SoapUITestCaseRunner extends AbstractSoapUIRunner implements TestRunListener
68  {
69  	public static final String TITLE = "soapUI " + SoapUI.SOAPUI_VERSION + " TestCase Runner";
70  	
71  	private String testSuite;
72  	private String testCase;
73  	private List<WsdlMessageAssertion> assertions = new ArrayList<WsdlMessageAssertion>();
74  	private Map<WsdlMessageAssertion,WsdlTestStepResult> assertionResults = new HashMap<WsdlMessageAssertion,WsdlTestStepResult>();
75  	private List<TestCase> runningTests = new ArrayList<TestCase>();
76  	private List<TestCase> failedTests = new ArrayList<TestCase>();
77  	private String endpoint;
78  	private String domain;
79  	private String password;
80  	private String username;
81  	private String host;
82  	
83  	private int testSuiteCount;
84  	private int testCaseCount;
85  	private int testStepCount;
86  	private int testAssertionCount;
87  
88  	private boolean printReport;
89  	private String outputFolder;
90  	private boolean exportAll;
91  	private boolean junitReport;
92  	private int exportCount;
93  	private JUnitReportCollector reportCollector;
94  	private String wssPasswordType;
95  	private WsdlProject project;
96  	
97  	/***
98  	 * Runs the tests in the specified soapUI project file, see soapUI xdocs for details.
99  	 * 
100 	 * @param args
101 	 * @throws Exception
102 	 */
103 
104 	@SuppressWarnings("static-access")
105 	public static void main( String [] args) throws Exception
106 	{
107 		new SoapUITestCaseRunner().runFromCommandLine( args );
108 	}
109 	
110 	protected boolean processCommandLine( CommandLine cmd )
111 	{
112 		if( cmd.hasOption( "e"))
113 			setEndpoint( cmd.getOptionValue( "e" ) );
114 		
115 		if( cmd.hasOption( "s"))
116 			setTestSuite( getCommandLineOptionSubstSpace( cmd, "s") );
117 
118 		if( cmd.hasOption( "c"))
119 			setTestCase( getCommandLineOptionSubstSpace( cmd, "c") );
120 
121 		if( cmd.hasOption( "u"))
122 			setUsername( cmd.getOptionValue( "u") );
123 
124 		if( cmd.hasOption( "p"))
125 			setPassword( cmd.getOptionValue( "p") );
126 		
127 		if( cmd.hasOption( "w"))
128 			setWssPasswordType( cmd.getOptionValue( "w") );
129 
130 		if( cmd.hasOption( "d"))
131 			setDomain( cmd.getOptionValue( "d") );
132 
133 		if( cmd.hasOption( "h"))
134 			setHost( cmd.getOptionValue( "h") );
135 		
136 		if( cmd.hasOption( "f"))
137 			setOutputFolder( getCommandLineOptionSubstSpace( cmd, "f") );
138 		
139 		if( cmd.hasOption( "t"))
140 			setSettingsFile( getCommandLineOptionSubstSpace( cmd, "t" ));
141 		
142 		setEnableUI( cmd.hasOption( "i") );
143 		setPrintReport( cmd.hasOption( "r" ) );
144 		setExportAll( cmd.hasOption( "a" ) );
145 		setJUnitReport( cmd.hasOption( "j" ) );
146 		
147 		return true;
148 	}
149 
150 	protected SoapUIOptions initCommandLineOptions()
151 	{
152 		SoapUIOptions options = new SoapUIOptions( "testrunner" );
153 		options.addOption( "e", true, "Sets the endpoint" );
154 		options.addOption( "s", true, "Sets the testsuite" );
155 		options.addOption( "c", true, "Sets the testcase" );
156 		options.addOption( "u", true, "Sets the username" );
157 		options.addOption( "p", true, "Sets the password" );
158 		options.addOption( "w", true, "Sets the WSS password type, either 'Text' or 'Digest'" );
159 		options.addOption( "i", false, "Enables Swing UI for scripts" );
160 		options.addOption( "d", true, "Sets the domain" );
161 		options.addOption( "h", true, "Sets the host" );
162 		options.addOption( "r", false, "Prints a small summary report" );
163 		options.addOption( "f", true, "Sets the output folder to export results to" );
164 		options.addOption( "j", false, "Sets the output to include JUnit XML reports" );
165 		options.addOption( "a", false, "Turns on exporting of all results" );
166 		options.addOption( "t", true, "Sets the soapui-settings.xml file to use" );
167 		return options;
168 	}
169 
170 	/***
171 	 * Add console appender to groovy log
172 	 */
173 	
174 	
175 	public void setExportAll(boolean exportAll)
176 	{
177 		this.exportAll = exportAll;
178 	}
179 
180 	public void setJUnitReport(boolean junitReport)
181 	{
182 		this.junitReport = junitReport;
183 		if (junitReport)
184 			reportCollector = new JUnitReportCollector();
185 	}
186 	
187 	public void setOutputFolder(String outputFolder)
188 	{
189 		this.outputFolder = outputFolder;
190 	}
191 	
192 	public SoapUITestCaseRunner()
193 	{
194 		super( SoapUITestCaseRunner.TITLE  );
195 	}
196 	
197 	public SoapUITestCaseRunner( String title )
198 	{
199 		super( title  );
200 	}
201 	
202 	/***
203 	 * Controls if a short test summary should be printed after the test runs
204 	 * 
205 	 * @param printReport a flag controlling if a summary should be printed
206 	 */
207 	
208 	public void setPrintReport(boolean printReport)
209 	{
210 		this.printReport = printReport;
211 	}
212 
213 	/***
214 	 * Sets the host to use by all test-requests, the existing endpoint port and path will be used
215 	 * 
216 	 * @param host the host to use by all requests
217 	 */
218 
219 	public void setHost(String host)
220 	{
221 		this.host = host;
222 	}
223 
224 	/***
225 	 * Sets the domain to use for any authentications
226 	 * 
227 	 * @param domain the domain to use for any authentications
228 	 */
229 	
230 	public void setDomain(String domain)
231 	{
232 		this.domain = domain;
233 	}
234 
235 	/***
236 	 * Sets the password to use for any authentications
237 	 * 
238 	 * @param domain the password to use for any authentications
239 	 */
240 	
241 	public void setPassword(String password)
242 	{
243 		this.password = password;
244 	}
245 	
246 	/***
247 	 * Sets the WSS password-type to use for any authentications. Setting this will result
248 	 * in the addition of WS-Security UsernamePassword tokens to any outgoing request containing
249 	 * the specified username and password.
250 	 * 
251 	 * @param wssPasswordType the wss-password type to use, either 'Text' or 'Digest'
252 	 */
253 	
254 	public void setWssPasswordType( String wssPasswordType )
255 	{
256 		this.wssPasswordType = wssPasswordType;
257 	}
258 
259 	/***
260 	 * Sets the username to use for any authentications
261 	 * 
262 	 * @param domain the username to use for any authentications
263 	 */
264 	
265 	public void setUsername(String username)
266 	{
267 		this.username = username;
268 	}
269 	
270 	/***
271 	 * Runs the testcases as configured with setXXX methods
272 	 * 
273 	 * @throws Exception thrown if any tests fail
274 	 */
275 
276 	public boolean runRunner() throws Exception
277 	{
278 		initGroovyLog();
279 		
280 		if( outputFolder != null )
281 		{
282 			//	ensure folder exists
283 			File folder = new File( outputFolder );
284 			if( !folder.exists())
285 				folder.mkdirs();
286 		}
287 		
288 		assertions.clear();
289 		
290 		String projectFile = getProjectFile();
291 		
292 		project = new WsdlProject( projectFile );
293 		if( project.isDisabled() )
294 			throw new Exception( "Failed to load soapUI project file [" + projectFile + "]" );
295 		
296 		log.info( "Running soapUI tests in project [" + project.getName() + "]" );
297 		
298 		long startTime = System.nanoTime();
299 		
300 		// start by listening to all testcases.. (since one testcase can call another)
301 		for( int c = 0; c < project.getTestSuiteCount(); c++ )
302 		{
303 			TestSuite suite = project.getTestSuiteAt( c );
304 			for( int i = 0; i < suite.getTestCaseCount(); i++ )
305 			{
306 				suite.getTestCaseAt( i ).addTestRunListener( this );
307 				if (junitReport)
308 					suite.getTestCaseAt( i ).addTestRunListener(reportCollector);
309 			}
310 		}
311 		
312 		// now run tests..
313 		for( int c = 0; c < project.getTestSuiteCount(); c++ )
314 		{
315 			WsdlTestSuite ts = project.getTestSuiteAt( c );
316 			if( !ts.isDisabled() && (testSuite == null || ts.getName().equalsIgnoreCase( testSuite )))
317 			{
318 				runSuite( ts);
319 				testSuiteCount++;
320 				
321 				//	wait for tests to finish if running in parallell mode
322 				if( !runningTests.isEmpty() )
323 					log.info( "Waiting for " + runningTests.size() + " tests to finish" );
324 				
325 				while( !runningTests.isEmpty() )
326 				{
327 					Thread.sleep( 100 );
328 				}
329 			}
330 		}
331 		
332 		long timeTaken = (System.nanoTime()-startTime)/1000000;
333 		
334 		if( printReport )
335 		{
336 			printReport( timeTaken );
337 		}
338 		
339 		if (junitReport) {
340 			exportJUnitReports( reportCollector, outputFolder );
341 		}
342 		
343 		if( assertions.size() > 0 || failedTests.size() > 0 )
344 		{
345 			throwFailureException();
346 		}
347 		
348 		return true;
349 	}
350 
351 	protected void throwFailureException() throws Exception
352 	{
353 		StringBuffer buf = new StringBuffer();
354 		
355 		for( int c = 0; c < assertions.size(); c++ )
356 		{
357 			WsdlMessageAssertion assertion = assertions.get( c );
358 			Assertable assertable =  assertion.getAssertable();
359 			if( assertable instanceof WsdlTestStep)
360 				failedTests.remove( ((WsdlTestStep)assertable).getTestCase() );
361 			
362 			buf.append( assertion.getName() + " in [" + assertable.getModelItem().getName() + "] failed;\n" );
363 			buf.append( Arrays.toString( assertion.getErrors() ) + "\n" );
364 			
365 			WsdlTestStepResult result = assertionResults.get( assertion  );
366 			StringWriter stringWriter = new StringWriter();
367 			PrintWriter writer = new PrintWriter( stringWriter );
368 			result.writeTo( writer );
369 			buf.append( stringWriter.toString() );
370 		}
371 
372 		while( !failedTests.isEmpty() )
373 		{
374 			buf.append( "TestCase [" + failedTests.remove( 0 ).getName() + "] failed without assertions\n" );
375 		}
376 		
377 		throw new Exception( buf.toString() );
378 	}
379 
380 	public void exportJUnitReports( JUnitReportCollector collector, String folder ) throws Exception
381 	{
382 		collector.saveReports(folder == null ? "" : folder);
383 	}
384 
385 	public void printReport( long timeTaken )
386 	{
387 		System.out.println();
388 		System.out.println( "SoapUI " + SoapUI.SOAPUI_VERSION + " TestCaseRunner Summary" );
389 		System.out.println( "-----------------------------" );
390 		System.out.println( "Time Taken: " + timeTaken + "ms" );
391 		System.out.println( "Total TestSuites: " + testSuiteCount );
392 		System.out.println( "Total TestCases: " + testCaseCount + " (" + failedTests.size() + " failed)");
393 		System.out.println( "Total TestSteps: " + testStepCount );
394 		System.out.println( "Total Request Assertions: " + testAssertionCount );
395 		System.out.println( "Total Failed Assertions: " + assertions.size() );
396 		System.out.println( "Total Exported Results: " + exportCount );
397 	}
398 
399 	/***
400 	 * Run tests in the specified TestSuite
401 	 *
402 	 * @param suite the TestSuite to run
403 	 */
404 	
405 	public void runSuite(WsdlTestSuite suite)
406 	{
407 		log.info(( "Running soapUI suite [" + suite.getName() + "], runType = " + suite.getRunType()));
408 		PropertyExpansionContext context = new DefaultPropertyExpansionContext( suite );
409 		long start = System.currentTimeMillis();
410 		
411 		try
412 		{
413 			suite.runSetupScript( context );
414 			for( int c = 0; c < suite.getTestCaseCount(); c++ )
415 			{
416 				WsdlTestCase tc = suite.getTestCaseAt( c );
417 				
418 				String name = tc.getName();
419 				if( testCase == null || name.equalsIgnoreCase( testCase ))
420 				{
421 					if( !tc.isDisabled() )
422 					{
423 						runTestCase( tc);
424 					}
425 					else
426 					{
427 						log.info( "Skipping disabled testcase [" + name + "]");
428 					}
429 				}
430 				else
431 				{
432 					log.info( "Skipping testcase [" + name + "], filter is [" + testCase + "]");
433 				}
434 			}
435 		}
436 		catch( Exception e )
437 		{
438 			e.printStackTrace();
439 		}
440 		finally
441 		{
442 			try
443 			{
444 				suite.runTearDownScript( context );
445 			}
446 			catch( Exception e )
447 			{
448 				e.printStackTrace();
449 			}
450 		}
451 		
452 		log.info( "soapUI suite [" + suite.getName() + "] finished in " + (System.currentTimeMillis()-start) + "ms" );
453 	}
454 
455 	/***
456 	 * Runs the specified TestCase
457 	 * 
458 	 * @param testCase the testcase to run
459 	 */
460 	
461 	private void runTestCase(TestCase testCase)
462 	{
463 		runningTests.add( testCase );
464 		testCase.run( new StringToObjectMap(),	testCase.getTestSuite().getRunType() == TestSuiteRunType.PARALLEL );
465 	}
466 
467 	/***
468 	 * Sets the testcase to run
469 	 * 
470 	 * @param testCase the testcase to run
471 	 */
472 	
473 	public void setTestCase(String testCase)
474 	{
475       this.testCase = testCase;
476 	}
477 	
478 	/***
479 	 * Sets the endpoint to use for all test requests
480 	 * 
481 	 * @param endpoint the endpoint to use for all test requests
482 	 */
483 	
484 	public void setEndpoint(String endpoint)
485 	{
486 		this.endpoint = endpoint.trim();
487 	}
488 	
489 	/***
490 	 * Sets the TestSuite to run. If not set all TestSuites in the specified project file are run
491 	 * 
492 	 * @param testSuite the testSuite to run.
493 	 */
494 
495 	public void setTestSuite(String testSuite)
496 	{
497 		this.testSuite = testSuite;
498 	}
499 	
500 	public void beforeRun(TestRunner testRunner, TestRunContext runContext)
501 	{
502 		log.info( "Running soapUI testcase [" + testRunner.getTestCase().getName() + "]");
503 	}
504 
505 	public void beforeStep(TestRunner testRunner, TestRunContext runContext)
506 	{
507 		TestStep currentStep = runContext.getCurrentStep();
508 		log.info( "running step [" + currentStep.getName() + "]" );
509 		
510 		if( currentStep instanceof WsdlTestRequestStep )
511 		{
512 			WsdlTestRequestStep requestStep = (WsdlTestRequestStep) currentStep;
513 			if( endpoint != null && endpoint.length() > 0 )
514 			{
515 				requestStep.getTestRequest().setEndpoint( endpoint );
516 			}
517 			
518 			if( host != null && host.length() > 0 )
519 			{
520 				try
521 				{
522 					String ep = Tools.replaceHost( requestStep.getTestRequest().getEndpoint(), host );
523 					requestStep.getTestRequest().setEndpoint( ep );
524 				}
525 				catch (Exception e)
526 				{
527 					log.error( "Failed to set host on endpoint", e );
528 				}				
529 			}
530 
531 			if( username != null && username.length() > 0 )
532 			{
533 				requestStep.getTestRequest().setUsername( username );
534 			}
535 			
536 			if( password != null && password.length() > 0 )
537 			{
538 				requestStep.getTestRequest().setPassword( password );
539 			}
540 			
541 			if( domain != null && domain.length() > 0 )
542 			{
543 				requestStep.getTestRequest().setDomain( domain );
544 			}
545 			
546 			if( wssPasswordType != null && wssPasswordType.length() > 0 )
547 			{
548 				requestStep.getTestRequest().setWssPasswordType( wssPasswordType.equals( "Digest" ) ? 
549 							WsdlTestRequest.PW_TYPE_DIGEST : WsdlTestRequest.PW_TYPE_TEXT );
550 			}
551 		}
552 	}
553 
554 	public void afterStep(TestRunner testRunner, TestRunContext runContext, TestStepResult result )
555 	{
556 		TestStep currentStep = runContext.getCurrentStep();
557 		
558 		if( currentStep instanceof WsdlTestRequestStep )
559 		{
560 			WsdlTestRequestStep requestStep = (WsdlTestRequestStep) currentStep;
561 			for( int c = 0; c < requestStep.getAssertionCount(); c++ )
562 			{
563 				WsdlMessageAssertion assertion = requestStep.getAssertionAt( c );
564 				log.info( "Assertion [" + assertion.getName() + "] has status " + assertion.getStatus());
565 				if( assertion.getStatus() == AssertionStatus.FAILED )
566 				{
567 					for( AssertionError error : assertion.getErrors())
568 						log.error( "ASSERTION FAILED -> " + error.getMessage());
569 					
570 					assertions.add( assertion );
571 					assertionResults.put( assertion, ( WsdlTestStepResult ) result );
572 				}
573 				
574 				testAssertionCount++;
575 			}
576 		}	
577 		
578 		String countPropertyName = currentStep.getName() + " run count";
579 		Long count = (Long) runContext.getProperty( countPropertyName );
580 		if( count == null )
581 		{
582 			count = new Long( 0 );
583 		}
584 
585 		runContext.setProperty( countPropertyName, new Long( count.longValue()+1 ) );
586 		
587 		if( result.getStatus() == TestStepStatus.FAILED || exportAll )
588 		{
589 			try
590 			{
591 				String nameBase = currentStep.getTestCase().getTestSuite().getName() + "-"
592 														  + currentStep.getTestCase().getName() + "-" + currentStep.getName() + "-" +	
593 														  count.longValue() + "-" + result.getStatus();
594 				
595 				String fileName = nameBase + ".txt";
596 				if( outputFolder != null )
597 				{
598 					fileName = outputFolder + File.separator + fileName;
599 				}
600 				
601 				if( result.getStatus() == TestStepStatus.FAILED )
602 					log.error( currentStep.getName() + " failed, exporting to [" + fileName + "]" );
603 				
604 				PrintWriter writer = new PrintWriter(fileName);
605 				result.writeTo(writer);
606 				writer.close();
607 				
608 				// write attachments
609 				if( result instanceof WsdlTestRequestStepResult )
610 				{
611 					Attachment [] attachments = ((WsdlTestRequestStepResult)result).getResponseAttachments();
612 					if( attachments != null && attachments.length > 0 )
613 					{
614 						for( int c = 0; c < attachments.length; c++ )
615 						{
616 							fileName = nameBase + "-attachment-" + (c+1) + ".";
617 							
618 							Attachment attachment = attachments[c];
619 							String contentType = attachment.getContentType();
620 							if( !"application/octet-stream".equals( contentType ) && contentType != null && contentType.indexOf( '/' ) != -1 )
621 							{
622 								fileName += contentType.substring( contentType.lastIndexOf( '/' )+1 );
623 							}
624 							else
625 							{
626 								fileName += "dat";
627 							}
628 							
629 							if( outputFolder != null )
630 							{
631 								fileName = outputFolder + File.separator + fileName;
632 							}
633 							
634 							FileOutputStream outFile = new FileOutputStream( fileName );
635 							Tools.writeAll( outFile, attachment.getInputStream() );
636 							outFile.close();
637 						}
638 					}
639 				}
640 				
641 				exportCount++;
642 			}
643 			catch (Exception e)
644 			{
645 				log.error( "Error saving failed result: " + e, e );
646 			}			
647 		}
648 		
649 		testStepCount++;
650 	}
651 
652 	public void afterRun(TestRunner testRunner, TestRunContext runContext)
653 	{
654 		log.info( "Finished running soapUI testcase [" + testRunner.getTestCase().getName() + "], time taken: " + 
655 				testRunner.getTimeTaken() + "ms, status: " + testRunner.getStatus() );
656 		
657 		if( testRunner.getStatus() == Status.FAILED )
658 		{
659 			failedTests.add( testRunner.getTestCase() );
660 		}
661 		
662 		runningTests.remove( testRunner.getTestCase() );
663 		
664 		testCaseCount++;
665 	}
666 
667 	protected WsdlProject getProject()
668 	{
669 		return project;
670 	}
671 }