View Javadoc

1   /*
2    *  soapUI, copyright (C) 2006 eviware.com 
3    *
4    *  soapUI is free software; you can redistribute it and/or modify it under the 
5    *  terms of the GNU Lesser General Public License as published by the Free Software Foundation; 
6    *  either version 2.1 of the License, or (at your option) any later version.
7    *
8    *  soapUI is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without 
9    *  even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 
10   *  See the GNU Lesser General Public License for more details at gnu.org.
11   */
12  
13  package com.eviware.soapui.tools;
14  
15  import java.io.File;
16  import java.io.OutputStreamWriter;
17  import java.io.PrintWriter;
18  import java.io.StringWriter;
19  import java.util.ArrayList;
20  import java.util.Arrays;
21  import java.util.HashMap;
22  import java.util.List;
23  import java.util.Map;
24  
25  import org.apache.commons.cli.CommandLine;
26  import org.apache.commons.cli.CommandLineParser;
27  import org.apache.commons.cli.HelpFormatter;
28  import org.apache.commons.cli.Options;
29  import org.apache.commons.cli.PosixParser;
30  import org.apache.log4j.ConsoleAppender;
31  import org.apache.log4j.Logger;
32  import org.apache.log4j.PatternLayout;
33  
34  import com.eviware.soapui.SoapUI;
35  import com.eviware.soapui.impl.wsdl.WsdlProject;
36  import com.eviware.soapui.impl.wsdl.panels.support.assertions.Assertable.AssertionStatus;
37  import com.eviware.soapui.impl.wsdl.teststeps.WsdlMessageAssertion;
38  import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestRequest;
39  import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestRequestStep;
40  import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestStepResult;
41  import com.eviware.soapui.model.support.PropertiesMap;
42  import com.eviware.soapui.model.testsuite.TestCase;
43  import com.eviware.soapui.model.testsuite.TestRunContext;
44  import com.eviware.soapui.model.testsuite.TestRunListener;
45  import com.eviware.soapui.model.testsuite.TestRunner;
46  import com.eviware.soapui.model.testsuite.TestStep;
47  import com.eviware.soapui.model.testsuite.TestStepResult;
48  import com.eviware.soapui.model.testsuite.TestSuite;
49  import com.eviware.soapui.model.testsuite.TestRunner.Status;
50  import com.eviware.soapui.model.testsuite.TestStepResult.TestStepStatus;
51  import com.eviware.soapui.model.testsuite.TestSuite.TestSuiteRunType;
52  import com.eviware.soapui.monitor.TestMonitor;
53  import com.eviware.soapui.report.JUnitReportCollector;
54  
55  /***
56   * Standalone test-runner used from maven-plugin, can also be used from command-line (see xdocs) or
57   * directly from other classes.
58   * <p>
59   * For standalone usage, set the project file (with setProjectFile) and other desired properties before
60   * calling run</p> 
61   * 
62   * @author Ole.Matzura
63   */
64  
65  public class SoapUITestCaseRunner implements TestRunListener
66  {
67  	private final static Logger log = Logger.getLogger( SoapUITestCaseRunner.class );
68  	
69  	private String projectFile;
70  	private String testSuite;
71  	private String testCase;
72  	private List<WsdlMessageAssertion> assertions = new ArrayList<WsdlMessageAssertion>();
73  	private Map<WsdlMessageAssertion,WsdlTestStepResult> assertionResults = new HashMap<WsdlMessageAssertion,WsdlTestStepResult>();
74  	private List<TestCase> runningTests = new ArrayList<TestCase>();
75  	private List<TestCase> failedTests = new ArrayList<TestCase>();
76  	private String endpoint;
77  	private String domain;
78  	private String password;
79  	private String username;
80  	private String host;
81  	
82  	private int testSuiteCount;
83  	private int testCaseCount;
84  	private int testStepCount;
85  	private int testAssertionCount;
86  
87  	private boolean printReport;
88  	private String outputFolder;
89  	private boolean exportAll;
90  	private boolean junitReport;
91  	private int exportCount;
92  	private JUnitReportCollector reportCollector;
93  	private String wssPasswordType;
94  	
95  	public static String TITLE = "soapUI " + SoapUI.SOAPUI_VERSION + " TestCase Runner";
96  	
97  	/***
98  	 * Runs the tests in the specified soapUI project file, see soapUI xdocs for details.
99  	 * 
100 	 * @param args
101 	 * @throws Exception
102 	 */
103 
104 	@SuppressWarnings("static-access")
105 	public static void main( String [] args) throws Exception
106 	{
107 		System.out.println( TITLE );
108 		SoapUITestCaseRunner runner = new SoapUITestCaseRunner();
109 		
110 		Options options = new Options();
111 		options.addOption( "e", true, "Sets the endpoint" );
112 		options.addOption( "s", true, "Sets the testsuite" );
113 		options.addOption( "c", true, "Sets the testcase" );
114 		options.addOption( "u", true, "Sets the username" );
115 		options.addOption( "p", true, "Sets the password" );
116 		options.addOption( "w", true, "Sets the WSS password type, either 'Text' or 'Digest'" );
117 		options.addOption( "d", true, "Sets the domain" );
118 		options.addOption( "h", true, "Sets the host" );
119 		options.addOption( "r", false, "Prints a small summary report" );
120 		options.addOption( "f", true, "Sets the output folder to export results to" );
121 		options.addOption( "j", false, "Sets the output to include JUnit XML reports" );
122 		options.addOption( "a", false, "Turns on exporting of all results" );
123 		options.addOption( "t", true, "Sets the soapui-settings.xml file to use" );
124 		
125 		CommandLineParser parser = new PosixParser();
126 		CommandLine cmd = parser.parse( options, args);
127 		
128 		String[] args2 = cmd.getArgs();
129 		if( args2.length != 1 )
130 		{
131 			HelpFormatter formatter = new HelpFormatter();
132 			formatter.printHelp( "testcaserunner [options] <soapui-project-file>", options );
133 
134 			System.err.println( "Missing soapUI project file.." );
135 			return;
136 		}
137 		
138 		runner.setProjectFile( args2[0] );
139 		
140 		if( cmd.hasOption( "e"))
141 			runner.setEndpoint( cmd.getOptionValue( "e" ) );
142 		
143 		if( cmd.hasOption( "s"))
144 			runner.setTestSuite( cmd.getOptionValue( "s") );
145 
146 		if( cmd.hasOption( "c"))
147 			runner.setTestCase( cmd.getOptionValue( "c") );
148 
149 		if( cmd.hasOption( "u"))
150 			runner.setUsername( cmd.getOptionValue( "u") );
151 
152 		if( cmd.hasOption( "p"))
153 			runner.setPassword( cmd.getOptionValue( "p") );
154 		
155 		if( cmd.hasOption( "w"))
156 			runner.setWssPasswordType( cmd.getOptionValue( "w") );
157 
158 		if( cmd.hasOption( "d"))
159 			runner.setDomain( cmd.getOptionValue( "d") );
160 
161 		if( cmd.hasOption( "h"))
162 			runner.setHost( cmd.getOptionValue( "h") );
163 		
164 		if( cmd.hasOption( "f"))
165 			runner.setOutputFolder( cmd.getOptionValue( "f") );
166 		
167 		if( cmd.hasOption( "t"))
168 			SoapUI.initSettings( cmd.getOptionValue( "t" ));
169 		
170 		runner.setPrintReport( cmd.hasOption( "r" ) );
171 		runner.setExportAll( cmd.hasOption( "a" ) );
172 		runner.setJUnitReport( cmd.hasOption( "j" ) );
173 		
174 		try
175 		{
176 			SoapUI.loadExtLibs();
177 			initGroovyLogger();
178 			
179 			runner.run();
180 			System.exit( 0 );
181 		}
182 		catch (Exception e)
183 		{
184 			log.error( e.toString() );
185 			e.printStackTrace();
186 			System.exit( 1 );
187 		}		
188 	}
189 
190 	/***
191 	 * Add console appender to groovy log
192 	 */
193 	
194 	public static void initGroovyLogger()
195 	{
196 		Logger logger = Logger.getLogger( "groovy.log" );
197 		
198 		ConsoleAppender appender = new ConsoleAppender();
199 		appender.setWriter( new OutputStreamWriter( System.out ));
200 		appender.setLayout( new PatternLayout( "%d{ABSOLUTE} %-5p [%c{1}] %m%n") );
201 		logger.addAppender( appender);
202 	}
203 	
204 	public void setExportAll(boolean exportAll)
205 	{
206 		this.exportAll = exportAll;
207 	}
208 
209 	public void setJUnitReport(boolean junitReport)
210 	{
211 		this.junitReport = junitReport;
212 		if (junitReport)
213 			reportCollector = new JUnitReportCollector();
214 	}
215 	
216 	public void setOutputFolder(String outputFolder)
217 	{
218 		this.outputFolder = outputFolder;
219 	}
220 
221 	public SoapUITestCaseRunner()
222 	{
223 		SoapUI.setTestMonitor( new TestMonitor() );
224 	}
225 	
226 	/***
227 	 * Controls if a short test summary should be printed after the test runs
228 	 * 
229 	 * @param printReport a flag controlling if a summary should be printed
230 	 */
231 	
232 	public void setPrintReport(boolean printReport)
233 	{
234 		this.printReport = printReport;
235 	}
236 
237 	/***
238 	 * Sets the host to use by all test-requests, the existing endpoint port and path will be used
239 	 * 
240 	 * @param host the host to use by all requests
241 	 */
242 
243 	public void setHost(String host)
244 	{
245 		this.host = host;
246 	}
247 
248 	/***
249 	 * Sets the domain to use for any authentications
250 	 * 
251 	 * @param domain the domain to use for any authentications
252 	 */
253 	
254 	public void setDomain(String domain)
255 	{
256 		log.info( "Setting domain to [" + domain + "]" );
257 		this.domain = domain;
258 	}
259 
260 	/***
261 	 * Sets the password to use for any authentications
262 	 * 
263 	 * @param domain the password to use for any authentications
264 	 */
265 	
266 	public void setPassword(String password)
267 	{
268 		log.info( "Setting password to [" + password + "]" );
269 		this.password = password;
270 	}
271 	
272 	/***
273 	 * Sets the WSS password-type to use for any authentications. Setting this will result
274 	 * in the addition of WS-Security UsernamePassword tokens to any outgoing request containing
275 	 * the specified username and password.
276 	 * 
277 	 * @param wssPasswordType the wss-password type to use, either 'Text' or 'Digest'
278 	 */
279 	
280 	public void setWssPasswordType( String wssPasswordType )
281 	{
282 		this.wssPasswordType = wssPasswordType;
283 	}
284 
285 	/***
286 	 * Sets the username to use for any authentications
287 	 * 
288 	 * @param domain the username to use for any authentications
289 	 */
290 	
291 	public void setUsername(String username)
292 	{
293 		log.info( "Setting username to [" + username + "]" );
294 		this.username = username;
295 	}
296 	
297 	/***
298 	 * Runs the testcases as configured with setXXX methods
299 	 * 
300 	 * @throws Exception thrown if any tests fail
301 	 */
302 
303 	public void run() throws Exception
304 	{
305 		assertions.clear();
306 		
307 		if( !new File( projectFile ).exists() )
308 			throw new Exception( "soapUI project file [" + projectFile + "] not found" );
309 		
310 		WsdlProject project = new WsdlProject( projectFile, null );
311 		log.info( "Running soapUI tests in project [" + project.getName() + "]" );
312 		
313 		long startTime = System.nanoTime();
314 		
315 		// start by listening to all testcases.. (since one testcase can call another)
316 		for( int c = 0; c < project.getTestSuiteCount(); c++ )
317 		{
318 			TestSuite suite = project.getTestSuiteAt( c );
319 			for( int i = 0; i < suite.getTestCaseCount(); i++ )
320 			{
321 				suite.getTestCaseAt( i ).addTestRunListener( this );
322 				if (junitReport)
323 					suite.getTestCaseAt( i ).addTestRunListener(reportCollector);
324 			}
325 		}
326 		
327 		// now run tests..
328 		for( int c = 0; c < project.getTestSuiteCount(); c++ )
329 		{
330 			if( testSuite == null || project.getTestSuiteAt( c ).getName().equalsIgnoreCase( testSuite ))
331 			{
332 				runSuite( project.getTestSuiteAt( c ));
333 				testSuiteCount++;
334 				
335 				//	wait for tests to finish if running in parallell mode
336 				if( !runningTests.isEmpty() )
337 					log.info( "Waiting for " + runningTests.size() + " tests to finish" );
338 				
339 				while( !runningTests.isEmpty() )
340 				{
341 					Thread.sleep( 100 );
342 				}
343 			}
344 		}
345 		
346 		long timeTaken = (System.nanoTime()-startTime)/1000000;
347 		
348 		if( printReport )
349 		{
350 			System.out.println();
351 			System.out.println( "SoapUI " + SoapUI.SOAPUI_VERSION + " TestCaseRunner Summary" );
352 			System.out.println( "-----------------------------" );
353 			System.out.println( "Time Taken: " + timeTaken + "ms" );
354 			System.out.println( "Total TestSuites: " + testSuiteCount );
355 			System.out.println( "Total TestCases: " + testCaseCount + " (" + failedTests.size() + " failed)");
356 			System.out.println( "Total TestSteps: " + testStepCount );
357 			System.out.println( "Total Request Assertions: " + testAssertionCount );
358 			System.out.println( "Total Failed Assertions: " + assertions.size() );
359 			System.out.println( "Total Exported Results: " + exportCount );
360 		}
361 		
362 		if (junitReport) {
363 			reportCollector.saveReports(outputFolder == null ? "" : outputFolder);
364 		}
365 		
366 		if( assertions.size() > 0 || failedTests.size() > 0 )
367 		{
368 			StringBuffer buf = new StringBuffer();
369 			
370 			for( int c = 0; c < assertions.size(); c++ )
371 			{
372 				WsdlMessageAssertion assertion = assertions.get( c );
373 				WsdlTestRequest testRequest = ((WsdlTestRequest)assertion.getAssertable());
374 				failedTests.remove( testRequest.getTestCase() );
375 				
376 				buf.append( assertion.getName() + " in [" + testRequest.getName() + "] failed;\n" );
377 				buf.append( Arrays.toString( assertion.getErrors() ) + "\n" );
378 				
379 				WsdlTestStepResult result = assertionResults.get( assertion  );
380 				StringWriter stringWriter = new StringWriter();
381 				PrintWriter writer = new PrintWriter( stringWriter );
382 				result.writeTo( writer );
383 				buf.append( stringWriter.toString() );
384 			}
385 
386 			while( !failedTests.isEmpty() )
387 			{
388 				buf.append( "TestCase [" + failedTests.remove( 0 ).getName() + "] failed without assertions\n" );
389 			}
390 			
391 			throw new Exception( buf.toString() );
392 		}
393 	}
394 
395 	/***
396 	 * Run tests in the specified TestSuite
397 	 *
398 	 * @param suite the TestSuite to run
399 	 */
400 	
401 	public void runSuite(TestSuite suite)
402 	{
403 		log.info(( "Running soapUI suite [" + suite.getName() + "], runType = " + suite.getRunType()));
404 		long start = System.currentTimeMillis();
405 		for( int c = 0; c < suite.getTestCaseCount(); c++ )
406 		{
407 			String name = suite.getTestCaseAt( c ).getName();
408 			if( testCase == null || name.equalsIgnoreCase( testCase ))
409 			{
410 				runTestCase( suite.getTestCaseAt( c ));
411 			}
412 			else
413 			{
414 				log.info( "Skipping testcase [" + name + "], filter is [" + testCase + "]");
415 			}
416 		}
417 		log.info( "soapUI suite [" + suite.getName() + "] finished in " + (System.currentTimeMillis()-start) + "ms" );
418 	}
419 
420 	/***
421 	 * Runs the specified TestCase
422 	 * 
423 	 * @param testCase the testcase to run
424 	 */
425 	
426 	private void runTestCase(TestCase testCase)
427 	{
428 		runningTests.add( testCase );
429 		testCase.run( PropertiesMap.EMPTY_MAP,	testCase.getTestSuite().getRunType() == TestSuiteRunType.PARALLEL );
430 	}
431 
432 	/***
433 	 * Sets the soapUI project file containing the tests to run
434 	 * 
435 	 * @param projectFile the soapUI project file containing the tests to run
436 	 */
437 	
438 	public void setProjectFile(String projectFile)
439 	{
440 		log.info( "setting projectFile to [" + projectFile + "]" );
441 		this.projectFile = projectFile;
442 	}
443 
444 	/***
445 	 * Sets the testcase to run
446 	 * 
447 	 * @param testCase the testcase to run
448 	 */
449 	
450 	public void setTestCase(String testCase)
451 	{
452 		log.info( "setting testCase to [" + testCase + "]" );
453       this.testCase = testCase;
454 	}
455 	
456 	/***
457 	 * Sets the endpoint to use for all test requests
458 	 * 
459 	 * @param endpoint the endpoint to use for all test requests
460 	 */
461 	
462 	public void setEndpoint(String endpoint)
463 	{
464 		log.info( "setting test endpoint to [" + endpoint+ "]" );
465 		this.endpoint = endpoint.trim();
466 	}
467 	
468 	/***
469 	 * Sets the TestSuite to run. If not set all TestSuites in the specified project file are run
470 	 * 
471 	 * @param testSuite the testSuite to run.
472 	 */
473 
474 	public void setTestSuite(String testSuite)
475 	{
476 	   log.info( "setting testSuite to [" + testSuite + "]" );
477 		this.testSuite = testSuite;
478 	}
479 	
480 	public void beforeRun(TestRunner testRunner, TestRunContext runContext)
481 	{
482 		log.info( "Running soapUI testcase [" + testRunner.getTestCase().getName() + "]");
483 	}
484 
485 	public void beforeStep(TestRunner testRunner, TestRunContext runContext)
486 	{
487 		TestStep currentStep = runContext.getCurrentStep();
488 		log.info( "running step [" + currentStep.getName() + "]" );
489 		
490 		if( currentStep instanceof WsdlTestRequestStep )
491 		{
492 			WsdlTestRequestStep requestStep = (WsdlTestRequestStep) currentStep;
493 			if( endpoint != null && endpoint.length() > 0 )
494 			{
495 				requestStep.getTestRequest().setEndpoint( endpoint );
496 			}
497 			
498 			if( host != null && host.length() > 0 )
499 			{
500 				try
501 				{
502 					String ep = replaceHost( requestStep.getTestRequest().getEndpoint(), host );
503 					requestStep.getTestRequest().setEndpoint( ep );
504 				}
505 				catch (Exception e)
506 				{
507 					log.error( "Failed to set host on endpoint", e );
508 				}				
509 			}
510 
511 			if( username != null && username.length() > 0 )
512 			{
513 				requestStep.getTestRequest().setUsername( username );
514 			}
515 			
516 			if( password != null && password.length() > 0 )
517 			{
518 				requestStep.getTestRequest().setPassword( password );
519 			}
520 			
521 			if( domain != null && domain.length() > 0 )
522 			{
523 				requestStep.getTestRequest().setDomain( domain );
524 			}
525 			
526 			if( wssPasswordType != null && wssPasswordType.length() > 0 )
527 			{
528 				requestStep.getTestRequest().setWssPasswordType( wssPasswordType.equals( "Digest" ) ? 
529 							WsdlTestRequest.PW_TYPE_DIGEST : WsdlTestRequest.PW_TYPE_TEXT );
530 			}
531 		}
532 	}
533 
534 	public void afterStep(TestRunner testRunner, TestRunContext runContext, TestStepResult result )
535 	{
536 		TestStep currentStep = runContext.getCurrentStep();
537 		
538 		if( currentStep instanceof WsdlTestRequestStep )
539 		{
540 			WsdlTestRequestStep requestStep = (WsdlTestRequestStep) currentStep;
541 			for( int c = 0; c < requestStep.getAssertionCount(); c++ )
542 			{
543 				WsdlMessageAssertion assertion = requestStep.getAssertionAt( c );
544 				log.info( "Assertion [" + assertion.getName() + "] has status " + assertion.getStatus());
545 				if( assertion.getStatus() == AssertionStatus.FAILED )
546 				{
547 					log.info( "ASSERTION FAILED -> " + assertion.getErrors());
548 					assertions.add( assertion );
549 					assertionResults.put( assertion, ( WsdlTestStepResult ) result );
550 				}
551 				
552 				testAssertionCount++;
553 			}
554 		}	
555 		
556 		String countPropertyName = currentStep.getName() + " run count";
557 		Long count = (Long) runContext.getProperty( countPropertyName );
558 		if( count == null )
559 		{
560 			count = new Long( 0 );
561 		}
562 
563 		runContext.setProperty( countPropertyName, new Long( count.longValue()+1 ) );
564 		
565 		if( result.getStatus() == TestStepStatus.FAILED || exportAll )
566 		{
567 			try
568 			{
569 				String fileName = currentStep.getTestCase().getTestSuite().getName() + "-"
570 										  + currentStep.getTestCase().getName() + "-" + currentStep.getName() + "-" +	
571 										  count.longValue() + "-" + result.getStatus() + ".txt";;
572 				
573 				if( outputFolder != null )
574 				{
575 					//	ensure folder exists
576 					File folder = new File( outputFolder );
577 					if( !folder.exists())
578 						folder.mkdirs();
579 					
580 					fileName = outputFolder + File.separator + fileName;
581 				}
582 				
583 				if( result.getStatus() == TestStepStatus.FAILED )
584 					log.error( currentStep.getName() + " failed, exporting to [" + fileName + "]" );
585 				
586 				PrintWriter writer = new PrintWriter(fileName);
587 				result.writeTo(writer);
588 				writer.close();
589 				
590 				exportCount++;
591 			}
592 			catch (Exception e)
593 			{
594 				log.error( "Error saving failed result: " + e, e );
595 			}			
596 		}
597 		
598 		testStepCount++;
599 	}
600 
601 	public void afterRun(TestRunner testRunner, TestRunContext runContext)
602 	{
603 		log.info( "Finished running soapUI testcase [" + testRunner.getTestCase().getName() + "], time taken: " + 
604 				testRunner.getTimeTaken() + "ms, status: " + testRunner.getStatus() );
605 		
606 		if( testRunner.getStatus() == Status.FAILED )
607 		{
608 			failedTests.add( testRunner.getTestCase() );
609 		}
610 		
611 		runningTests.remove( testRunner.getTestCase() );
612 		
613 		testCaseCount++;
614 	}
615 	
616 	/***
617 	 * Replaces the host part of the specified endpoint with the specified host
618 	 * 
619 	 * @param endpoint the endpoint to modify
620 	 * @param host the host to set
621 	 * @return the modified endpoint
622 	 */
623 	
624 	public static String replaceHost(String endpoint, String host)
625 	{
626 		int ix1 = endpoint.indexOf( "://" );
627 		if( ix1 < 0 )
628 			return endpoint;
629 		
630 		int ix2 = endpoint.indexOf( ":", ix1+3 );
631 		if( ix2 == -1 || host.indexOf( ":") > 0 )
632 		{
633 			ix2 = endpoint.indexOf( "/", ix1+3 );
634 			if( ix2 == ix1+3 )
635 				ix2 = -1;
636 		}
637 		
638 		return endpoint.substring( 0, ix1 )+ "://" + host + (ix2 == -1 ? "" : endpoint.substring(ix2));
639 	}
640 }