View Javadoc

1   /*
2    *  soapUI, copyright (C) 2004-2007 eviware.com 
3    *
4    *  soapUI is free software; you can redistribute it and/or modify it under the 
5    *  terms of version 2.1 of the GNU Lesser General Public License as published by 
6    *  the Free Software Foundation.
7    *
8    *  soapUI is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without 
9    *  even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 
10   *  See the GNU Lesser General Public License for more details at gnu.org.
11   */
12  
13  package com.eviware.soapui.tools;
14  
15  import java.io.File;
16  import java.io.OutputStreamWriter;
17  import java.io.PrintWriter;
18  import java.io.StringWriter;
19  import java.util.ArrayList;
20  import java.util.Arrays;
21  import java.util.HashMap;
22  import java.util.List;
23  import java.util.Map;
24  
25  import org.apache.commons.cli.CommandLine;
26  import org.apache.commons.cli.CommandLineParser;
27  import org.apache.commons.cli.HelpFormatter;
28  import org.apache.commons.cli.Options;
29  import org.apache.commons.cli.PosixParser;
30  import org.apache.log4j.ConsoleAppender;
31  import org.apache.log4j.Logger;
32  import org.apache.log4j.PatternLayout;
33  
34  import com.eviware.soapui.SoapUI;
35  import com.eviware.soapui.impl.wsdl.WsdlProject;
36  import com.eviware.soapui.impl.wsdl.panels.support.assertions.Assertable.AssertionStatus;
37  import com.eviware.soapui.impl.wsdl.teststeps.WsdlMessageAssertion;
38  import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestRequest;
39  import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestRequestStep;
40  import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestStepResult;
41  import com.eviware.soapui.model.support.PropertiesMap;
42  import com.eviware.soapui.model.testsuite.TestCase;
43  import com.eviware.soapui.model.testsuite.TestRunContext;
44  import com.eviware.soapui.model.testsuite.TestRunListener;
45  import com.eviware.soapui.model.testsuite.TestRunner;
46  import com.eviware.soapui.model.testsuite.TestStep;
47  import com.eviware.soapui.model.testsuite.TestStepResult;
48  import com.eviware.soapui.model.testsuite.TestSuite;
49  import com.eviware.soapui.model.testsuite.TestRunner.Status;
50  import com.eviware.soapui.model.testsuite.TestStepResult.TestStepStatus;
51  import com.eviware.soapui.model.testsuite.TestSuite.TestSuiteRunType;
52  import com.eviware.soapui.monitor.TestMonitor;
53  import com.eviware.soapui.report.JUnitReportCollector;
54  
55  /***
56   * Standalone test-runner used from maven-plugin, can also be used from command-line (see xdocs) or
57   * directly from other classes.
58   * <p>
59   * For standalone usage, set the project file (with setProjectFile) and other desired properties before
60   * calling run</p> 
61   * 
62   * @author Ole.Matzura
63   */
64  
65  public class SoapUITestCaseRunner implements TestRunListener
66  {
67  	private final static Logger log = Logger.getLogger( SoapUITestCaseRunner.class );
68  	
69  	private String projectFile;
70  	private String testSuite;
71  	private String testCase;
72  	private List<WsdlMessageAssertion> assertions = new ArrayList<WsdlMessageAssertion>();
73  	private Map<WsdlMessageAssertion,WsdlTestStepResult> assertionResults = new HashMap<WsdlMessageAssertion,WsdlTestStepResult>();
74  	private List<TestCase> runningTests = new ArrayList<TestCase>();
75  	private List<TestCase> failedTests = new ArrayList<TestCase>();
76  	private String endpoint;
77  	private String domain;
78  	private String password;
79  	private String username;
80  	private String host;
81  	
82  	private int testSuiteCount;
83  	private int testCaseCount;
84  	private int testStepCount;
85  	private int testAssertionCount;
86  
87  	private boolean printReport;
88  	private String outputFolder;
89  	private boolean exportAll;
90  	private boolean junitReport;
91  	private int exportCount;
92  	private JUnitReportCollector reportCollector;
93  	private String wssPasswordType;
94  	
95  	public static String TITLE = "soapUI " + SoapUI.SOAPUI_VERSION + " TestCase Runner";
96  	
97  	/***
98  	 * Runs the tests in the specified soapUI project file, see soapUI xdocs for details.
99  	 * 
100 	 * @param args
101 	 * @throws Exception
102 	 */
103 
104 	@SuppressWarnings("static-access")
105 	public static void main( String [] args) throws Exception
106 	{
107 		System.out.println( TITLE );
108 		SoapUI.initSoapUILog();
109 		SoapUITestCaseRunner runner = new SoapUITestCaseRunner();
110 		
111 		Options options = new Options();
112 		options.addOption( "e", true, "Sets the endpoint" );
113 		options.addOption( "s", true, "Sets the testsuite" );
114 		options.addOption( "c", true, "Sets the testcase" );
115 		options.addOption( "u", true, "Sets the username" );
116 		options.addOption( "p", true, "Sets the password" );
117 		options.addOption( "w", true, "Sets the WSS password type, either 'Text' or 'Digest'" );
118 		options.addOption( "d", true, "Sets the domain" );
119 		options.addOption( "h", true, "Sets the host" );
120 		options.addOption( "r", false, "Prints a small summary report" );
121 		options.addOption( "f", true, "Sets the output folder to export results to" );
122 		options.addOption( "j", false, "Sets the output to include JUnit XML reports" );
123 		options.addOption( "a", false, "Turns on exporting of all results" );
124 		options.addOption( "t", true, "Sets the soapui-settings.xml file to use" );
125 		
126 		CommandLineParser parser = new PosixParser();
127 		CommandLine cmd = parser.parse( options, args);
128 		
129 		String[] args2 = cmd.getArgs();
130 		if( args2.length != 1 )
131 		{
132 			HelpFormatter formatter = new HelpFormatter();
133 			formatter.printHelp( "testcaserunner [options] <soapui-project-file>", options );
134 
135 			System.err.println( "Missing soapUI project file.." );
136 			return;
137 		}
138 		
139 		runner.setProjectFile( args2[0] );
140 		
141 		if( cmd.hasOption( "e"))
142 			runner.setEndpoint( cmd.getOptionValue( "e" ) );
143 		
144 		if( cmd.hasOption( "s"))
145 			runner.setTestSuite( cmd.getOptionValue( "s") );
146 
147 		if( cmd.hasOption( "c"))
148 			runner.setTestCase( cmd.getOptionValue( "c") );
149 
150 		if( cmd.hasOption( "u"))
151 			runner.setUsername( cmd.getOptionValue( "u") );
152 
153 		if( cmd.hasOption( "p"))
154 			runner.setPassword( cmd.getOptionValue( "p") );
155 		
156 		if( cmd.hasOption( "w"))
157 			runner.setWssPasswordType( cmd.getOptionValue( "w") );
158 
159 		if( cmd.hasOption( "d"))
160 			runner.setDomain( cmd.getOptionValue( "d") );
161 
162 		if( cmd.hasOption( "h"))
163 			runner.setHost( cmd.getOptionValue( "h") );
164 		
165 		if( cmd.hasOption( "f"))
166 			runner.setOutputFolder( cmd.getOptionValue( "f") );
167 		
168 		if( cmd.hasOption( "t"))
169 			SoapUI.initSettings( cmd.getOptionValue( "t" ));
170 		
171 		runner.setPrintReport( cmd.hasOption( "r" ) );
172 		runner.setExportAll( cmd.hasOption( "a" ) );
173 		runner.setJUnitReport( cmd.hasOption( "j" ) );
174 		
175 		try
176 		{
177 			SoapUI.loadExtLibs();
178 			initGroovyLogger();
179 			
180 			runner.run();
181 			System.exit( 0 );
182 		}
183 		catch (Exception e)
184 		{
185 			log.error( e.toString() );
186 			e.printStackTrace();
187 			System.exit( 1 );
188 		}		
189 	}
190 
191 	/***
192 	 * Add console appender to groovy log
193 	 */
194 	
195 	public static void initGroovyLogger()
196 	{
197 		Logger logger = Logger.getLogger( "groovy.log" );
198 		
199 		ConsoleAppender appender = new ConsoleAppender();
200 		appender.setWriter( new OutputStreamWriter( System.out ));
201 		appender.setLayout( new PatternLayout( "%d{ABSOLUTE} %-5p [%c{1}] %m%n") );
202 		logger.addAppender( appender);
203 	}
204 	
205 	public void setExportAll(boolean exportAll)
206 	{
207 		this.exportAll = exportAll;
208 	}
209 
210 	public void setJUnitReport(boolean junitReport)
211 	{
212 		this.junitReport = junitReport;
213 		if (junitReport)
214 			reportCollector = new JUnitReportCollector();
215 	}
216 	
217 	public void setOutputFolder(String outputFolder)
218 	{
219 		this.outputFolder = outputFolder;
220 	}
221 
222 	public SoapUITestCaseRunner()
223 	{
224 		SoapUI.setTestMonitor( new TestMonitor() );
225 	}
226 	
227 	/***
228 	 * Controls if a short test summary should be printed after the test runs
229 	 * 
230 	 * @param printReport a flag controlling if a summary should be printed
231 	 */
232 	
233 	public void setPrintReport(boolean printReport)
234 	{
235 		this.printReport = printReport;
236 	}
237 
238 	/***
239 	 * Sets the host to use by all test-requests, the existing endpoint port and path will be used
240 	 * 
241 	 * @param host the host to use by all requests
242 	 */
243 
244 	public void setHost(String host)
245 	{
246 		this.host = host;
247 	}
248 
249 	/***
250 	 * Sets the domain to use for any authentications
251 	 * 
252 	 * @param domain the domain to use for any authentications
253 	 */
254 	
255 	public void setDomain(String domain)
256 	{
257 		log.info( "Setting domain to [" + domain + "]" );
258 		this.domain = domain;
259 	}
260 
261 	/***
262 	 * Sets the password to use for any authentications
263 	 * 
264 	 * @param domain the password to use for any authentications
265 	 */
266 	
267 	public void setPassword(String password)
268 	{
269 		log.info( "Setting password to [" + password + "]" );
270 		this.password = password;
271 	}
272 	
273 	/***
274 	 * Sets the WSS password-type to use for any authentications. Setting this will result
275 	 * in the addition of WS-Security UsernamePassword tokens to any outgoing request containing
276 	 * the specified username and password.
277 	 * 
278 	 * @param wssPasswordType the wss-password type to use, either 'Text' or 'Digest'
279 	 */
280 	
281 	public void setWssPasswordType( String wssPasswordType )
282 	{
283 		this.wssPasswordType = wssPasswordType;
284 	}
285 
286 	/***
287 	 * Sets the username to use for any authentications
288 	 * 
289 	 * @param domain the username to use for any authentications
290 	 */
291 	
292 	public void setUsername(String username)
293 	{
294 		log.info( "Setting username to [" + username + "]" );
295 		this.username = username;
296 	}
297 	
298 	/***
299 	 * Runs the testcases as configured with setXXX methods
300 	 * 
301 	 * @throws Exception thrown if any tests fail
302 	 */
303 
304 	public void run() throws Exception
305 	{
306 		assertions.clear();
307 		
308 		if( !new File( projectFile ).exists() )
309 			throw new Exception( "soapUI project file [" + projectFile + "] not found" );
310 		
311 		WsdlProject project = new WsdlProject( projectFile, null );
312 		log.info( "Running soapUI tests in project [" + project.getName() + "]" );
313 		
314 		long startTime = System.nanoTime();
315 		
316 		// start by listening to all testcases.. (since one testcase can call another)
317 		for( int c = 0; c < project.getTestSuiteCount(); c++ )
318 		{
319 			TestSuite suite = project.getTestSuiteAt( c );
320 			for( int i = 0; i < suite.getTestCaseCount(); i++ )
321 			{
322 				suite.getTestCaseAt( i ).addTestRunListener( this );
323 				if (junitReport)
324 					suite.getTestCaseAt( i ).addTestRunListener(reportCollector);
325 			}
326 		}
327 		
328 		// now run tests..
329 		for( int c = 0; c < project.getTestSuiteCount(); c++ )
330 		{
331 			if( testSuite == null || project.getTestSuiteAt( c ).getName().equalsIgnoreCase( testSuite ))
332 			{
333 				runSuite( project.getTestSuiteAt( c ));
334 				testSuiteCount++;
335 				
336 				//	wait for tests to finish if running in parallell mode
337 				if( !runningTests.isEmpty() )
338 					log.info( "Waiting for " + runningTests.size() + " tests to finish" );
339 				
340 				while( !runningTests.isEmpty() )
341 				{
342 					Thread.sleep( 100 );
343 				}
344 			}
345 		}
346 		
347 		long timeTaken = (System.nanoTime()-startTime)/1000000;
348 		
349 		if( printReport )
350 		{
351 			System.out.println();
352 			System.out.println( "SoapUI " + SoapUI.SOAPUI_VERSION + " TestCaseRunner Summary" );
353 			System.out.println( "-----------------------------" );
354 			System.out.println( "Time Taken: " + timeTaken + "ms" );
355 			System.out.println( "Total TestSuites: " + testSuiteCount );
356 			System.out.println( "Total TestCases: " + testCaseCount + " (" + failedTests.size() + " failed)");
357 			System.out.println( "Total TestSteps: " + testStepCount );
358 			System.out.println( "Total Request Assertions: " + testAssertionCount );
359 			System.out.println( "Total Failed Assertions: " + assertions.size() );
360 			System.out.println( "Total Exported Results: " + exportCount );
361 		}
362 		
363 		if (junitReport) {
364 			reportCollector.saveReports(outputFolder == null ? "" : outputFolder);
365 		}
366 		
367 		if( assertions.size() > 0 || failedTests.size() > 0 )
368 		{
369 			StringBuffer buf = new StringBuffer();
370 			
371 			for( int c = 0; c < assertions.size(); c++ )
372 			{
373 				WsdlMessageAssertion assertion = assertions.get( c );
374 				WsdlTestRequest testRequest = ((WsdlTestRequest)assertion.getAssertable());
375 				failedTests.remove( testRequest.getTestCase() );
376 				
377 				buf.append( assertion.getName() + " in [" + testRequest.getName() + "] failed;\n" );
378 				buf.append( Arrays.toString( assertion.getErrors() ) + "\n" );
379 				
380 				WsdlTestStepResult result = assertionResults.get( assertion  );
381 				StringWriter stringWriter = new StringWriter();
382 				PrintWriter writer = new PrintWriter( stringWriter );
383 				result.writeTo( writer );
384 				buf.append( stringWriter.toString() );
385 			}
386 
387 			while( !failedTests.isEmpty() )
388 			{
389 				buf.append( "TestCase [" + failedTests.remove( 0 ).getName() + "] failed without assertions\n" );
390 			}
391 			
392 			throw new Exception( buf.toString() );
393 		}
394 	}
395 
396 	/***
397 	 * Run tests in the specified TestSuite
398 	 *
399 	 * @param suite the TestSuite to run
400 	 */
401 	
402 	public void runSuite(TestSuite suite)
403 	{
404 		log.info(( "Running soapUI suite [" + suite.getName() + "], runType = " + suite.getRunType()));
405 		long start = System.currentTimeMillis();
406 		for( int c = 0; c < suite.getTestCaseCount(); c++ )
407 		{
408 			String name = suite.getTestCaseAt( c ).getName();
409 			if( testCase == null || name.equalsIgnoreCase( testCase ))
410 			{
411 				runTestCase( suite.getTestCaseAt( c ));
412 			}
413 			else
414 			{
415 				log.info( "Skipping testcase [" + name + "], filter is [" + testCase + "]");
416 			}
417 		}
418 		log.info( "soapUI suite [" + suite.getName() + "] finished in " + (System.currentTimeMillis()-start) + "ms" );
419 	}
420 
421 	/***
422 	 * Runs the specified TestCase
423 	 * 
424 	 * @param testCase the testcase to run
425 	 */
426 	
427 	private void runTestCase(TestCase testCase)
428 	{
429 		runningTests.add( testCase );
430 		testCase.run( PropertiesMap.EMPTY_MAP,	testCase.getTestSuite().getRunType() == TestSuiteRunType.PARALLEL );
431 	}
432 
433 	/***
434 	 * Sets the soapUI project file containing the tests to run
435 	 * 
436 	 * @param projectFile the soapUI project file containing the tests to run
437 	 */
438 	
439 	public void setProjectFile(String projectFile)
440 	{
441 		log.info( "setting projectFile to [" + projectFile + "]" );
442 		this.projectFile = projectFile;
443 	}
444 
445 	/***
446 	 * Sets the testcase to run
447 	 * 
448 	 * @param testCase the testcase to run
449 	 */
450 	
451 	public void setTestCase(String testCase)
452 	{
453 		log.info( "setting testCase to [" + testCase + "]" );
454       this.testCase = testCase;
455 	}
456 	
457 	/***
458 	 * Sets the endpoint to use for all test requests
459 	 * 
460 	 * @param endpoint the endpoint to use for all test requests
461 	 */
462 	
463 	public void setEndpoint(String endpoint)
464 	{
465 		log.info( "setting test endpoint to [" + endpoint+ "]" );
466 		this.endpoint = endpoint.trim();
467 	}
468 	
469 	/***
470 	 * Sets the TestSuite to run. If not set all TestSuites in the specified project file are run
471 	 * 
472 	 * @param testSuite the testSuite to run.
473 	 */
474 
475 	public void setTestSuite(String testSuite)
476 	{
477 	   log.info( "setting testSuite to [" + testSuite + "]" );
478 		this.testSuite = testSuite;
479 	}
480 	
481 	public void beforeRun(TestRunner testRunner, TestRunContext runContext)
482 	{
483 		log.info( "Running soapUI testcase [" + testRunner.getTestCase().getName() + "]");
484 	}
485 
486 	public void beforeStep(TestRunner testRunner, TestRunContext runContext)
487 	{
488 		TestStep currentStep = runContext.getCurrentStep();
489 		log.info( "running step [" + currentStep.getName() + "]" );
490 		
491 		if( currentStep instanceof WsdlTestRequestStep )
492 		{
493 			WsdlTestRequestStep requestStep = (WsdlTestRequestStep) currentStep;
494 			if( endpoint != null && endpoint.length() > 0 )
495 			{
496 				requestStep.getTestRequest().setEndpoint( endpoint );
497 			}
498 			
499 			if( host != null && host.length() > 0 )
500 			{
501 				try
502 				{
503 					String ep = replaceHost( requestStep.getTestRequest().getEndpoint(), host );
504 					requestStep.getTestRequest().setEndpoint( ep );
505 				}
506 				catch (Exception e)
507 				{
508 					log.error( "Failed to set host on endpoint", e );
509 				}				
510 			}
511 
512 			if( username != null && username.length() > 0 )
513 			{
514 				requestStep.getTestRequest().setUsername( username );
515 			}
516 			
517 			if( password != null && password.length() > 0 )
518 			{
519 				requestStep.getTestRequest().setPassword( password );
520 			}
521 			
522 			if( domain != null && domain.length() > 0 )
523 			{
524 				requestStep.getTestRequest().setDomain( domain );
525 			}
526 			
527 			if( wssPasswordType != null && wssPasswordType.length() > 0 )
528 			{
529 				requestStep.getTestRequest().setWssPasswordType( wssPasswordType.equals( "Digest" ) ? 
530 							WsdlTestRequest.PW_TYPE_DIGEST : WsdlTestRequest.PW_TYPE_TEXT );
531 			}
532 		}
533 	}
534 
535 	public void afterStep(TestRunner testRunner, TestRunContext runContext, TestStepResult result )
536 	{
537 		TestStep currentStep = runContext.getCurrentStep();
538 		
539 		if( currentStep instanceof WsdlTestRequestStep )
540 		{
541 			WsdlTestRequestStep requestStep = (WsdlTestRequestStep) currentStep;
542 			for( int c = 0; c < requestStep.getAssertionCount(); c++ )
543 			{
544 				WsdlMessageAssertion assertion = requestStep.getAssertionAt( c );
545 				log.info( "Assertion [" + assertion.getName() + "] has status " + assertion.getStatus());
546 				if( assertion.getStatus() == AssertionStatus.FAILED )
547 				{
548 					log.info( "ASSERTION FAILED -> " + assertion.getErrors());
549 					assertions.add( assertion );
550 					assertionResults.put( assertion, ( WsdlTestStepResult ) result );
551 				}
552 				
553 				testAssertionCount++;
554 			}
555 		}	
556 		
557 		String countPropertyName = currentStep.getName() + " run count";
558 		Long count = (Long) runContext.getProperty( countPropertyName );
559 		if( count == null )
560 		{
561 			count = new Long( 0 );
562 		}
563 
564 		runContext.setProperty( countPropertyName, new Long( count.longValue()+1 ) );
565 		
566 		if( result.getStatus() == TestStepStatus.FAILED || exportAll )
567 		{
568 			try
569 			{
570 				String fileName = currentStep.getTestCase().getTestSuite().getName() + "-"
571 										  + currentStep.getTestCase().getName() + "-" + currentStep.getName() + "-" +	
572 										  count.longValue() + "-" + result.getStatus() + ".txt";;
573 				
574 				if( outputFolder != null )
575 				{
576 					//	ensure folder exists
577 					File folder = new File( outputFolder );
578 					if( !folder.exists())
579 						folder.mkdirs();
580 					
581 					fileName = outputFolder + File.separator + fileName;
582 				}
583 				
584 				if( result.getStatus() == TestStepStatus.FAILED )
585 					log.error( currentStep.getName() + " failed, exporting to [" + fileName + "]" );
586 				
587 				PrintWriter writer = new PrintWriter(fileName);
588 				result.writeTo(writer);
589 				writer.close();
590 				
591 				exportCount++;
592 			}
593 			catch (Exception e)
594 			{
595 				log.error( "Error saving failed result: " + e, e );
596 			}			
597 		}
598 		
599 		testStepCount++;
600 	}
601 
602 	public void afterRun(TestRunner testRunner, TestRunContext runContext)
603 	{
604 		log.info( "Finished running soapUI testcase [" + testRunner.getTestCase().getName() + "], time taken: " + 
605 				testRunner.getTimeTaken() + "ms, status: " + testRunner.getStatus() );
606 		
607 		if( testRunner.getStatus() == Status.FAILED )
608 		{
609 			failedTests.add( testRunner.getTestCase() );
610 		}
611 		
612 		runningTests.remove( testRunner.getTestCase() );
613 		
614 		testCaseCount++;
615 	}
616 	
617 	/***
618 	 * Replaces the host part of the specified endpoint with the specified host
619 	 * 
620 	 * @param endpoint the endpoint to modify
621 	 * @param host the host to set
622 	 * @return the modified endpoint
623 	 */
624 	
625 	public static String replaceHost(String endpoint, String host)
626 	{
627 		int ix1 = endpoint.indexOf( "://" );
628 		if( ix1 < 0 )
629 			return endpoint;
630 		
631 		int ix2 = endpoint.indexOf( ":", ix1+3 );
632 		if( ix2 == -1 || host.indexOf( ":") > 0 )
633 		{
634 			ix2 = endpoint.indexOf( "/", ix1+3 );
635 			if( ix2 == ix1+3 )
636 				ix2 = -1;
637 		}
638 		
639 		return endpoint.substring( 0, ix1 )+ "://" + host + (ix2 == -1 ? "" : endpoint.substring(ix2));
640 	}
641 }