View Javadoc

1   /*
2    *  soapUI, copyright (C) 2004-2009 eviware.com 
3    *
4    *  soapUI is free software; you can redistribute it and/or modify it under the 
5    *  terms of version 2.1 of the GNU Lesser General Public License as published by 
6    *  the Free Software Foundation.
7    *
8    *  soapUI is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without 
9    *  even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 
10   *  See the GNU Lesser General Public License for more details at gnu.org.
11   */
12  
13  package com.eviware.soapui.tools;
14  
15  import java.io.File;
16  import java.io.FileOutputStream;
17  import java.io.PrintWriter;
18  import java.io.StringWriter;
19  import java.util.ArrayList;
20  import java.util.Arrays;
21  import java.util.HashMap;
22  import java.util.List;
23  import java.util.Map;
24  
25  import org.apache.commons.cli.CommandLine;
26  
27  import com.eviware.soapui.SoapUI;
28  import com.eviware.soapui.impl.wsdl.WsdlProject;
29  import com.eviware.soapui.impl.wsdl.WsdlTestSuite;
30  import com.eviware.soapui.impl.wsdl.testcase.WsdlProjectRunner;
31  import com.eviware.soapui.impl.wsdl.testcase.WsdlTestCase;
32  import com.eviware.soapui.impl.wsdl.testcase.WsdlTestCaseRunner;
33  import com.eviware.soapui.impl.wsdl.testcase.WsdlTestSuiteRunner;
34  import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestRequestStepResult;
35  import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestStep;
36  import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestStepResult;
37  import com.eviware.soapui.model.iface.Attachment;
38  import com.eviware.soapui.model.project.ProjectFactoryRegistry;
39  import com.eviware.soapui.model.support.ModelSupport;
40  import com.eviware.soapui.model.testsuite.Assertable;
41  import com.eviware.soapui.model.testsuite.AssertionError;
42  import com.eviware.soapui.model.testsuite.TestAssertion;
43  import com.eviware.soapui.model.testsuite.TestCase;
44  import com.eviware.soapui.model.testsuite.TestCaseRunContext;
45  import com.eviware.soapui.model.testsuite.TestCaseRunner;
46  import com.eviware.soapui.model.testsuite.TestStep;
47  import com.eviware.soapui.model.testsuite.TestStepResult;
48  import com.eviware.soapui.model.testsuite.TestSuite;
49  import com.eviware.soapui.model.testsuite.Assertable.AssertionStatus;
50  import com.eviware.soapui.model.testsuite.TestRunner.Status;
51  import com.eviware.soapui.model.testsuite.TestStepResult.TestStepStatus;
52  import com.eviware.soapui.report.JUnitReportCollector;
53  import com.eviware.soapui.support.StringUtils;
54  import com.eviware.soapui.support.Tools;
55  import com.eviware.soapui.support.types.StringToObjectMap;
56  
57  /***
58   * Standalone test-runner used from maven-plugin, can also be used from
59   * command-line (see xdocs) or directly from other classes.
60   * <p>
61   * For standalone usage, set the project file (with setProjectFile) and other
62   * desired properties before calling run
63   * </p>
64   * 
65   * @author Ole.Matzura
66   */
67  
68  public class SoapUITestCaseRunner extends AbstractSoapUITestRunner
69  {
70  	public static final String TITLE = "soapUI " + SoapUI.SOAPUI_VERSION + " TestCase Runner";
71  
72  	private String testSuite;
73  	private String testCase;
74  	private List<TestAssertion> assertions = new ArrayList<TestAssertion>();
75  	private Map<TestAssertion, WsdlTestStepResult> assertionResults = new HashMap<TestAssertion, WsdlTestStepResult>();
76  //	private List<TestCaseRunner> runningTests = new ArrayList<TestCaseRunner>();
77  	private List<TestCase> failedTests = new ArrayList<TestCase>();
78  
79  	private int testSuiteCount;
80  	private int testCaseCount;
81  	private int testStepCount;
82  	private int testAssertionCount;
83  
84  	private boolean printReport;
85  	private boolean exportAll;
86  	private boolean ignoreErrors;
87  	private boolean junitReport;
88  	private int exportCount;
89  	private JUnitReportCollector reportCollector;
90  //	private WsdlProject project;
91  	private String projectPassword;
92  	private boolean saveAfterRun;
93  
94  	/***
95  	 * Runs the tests in the specified soapUI project file, see soapUI xdocs for
96  	 * details.
97  	 * 
98  	 * @param args
99  	 * @throws Exception
100 	 */
101 
102 	public static void main( String[] args ) throws Exception
103 	{
104 		System.exit( new SoapUITestCaseRunner().runFromCommandLine( args ));
105 	}
106 
107 	protected boolean processCommandLine( CommandLine cmd )
108 	{
109 		if( cmd.hasOption( "e" ) )
110 			setEndpoint( cmd.getOptionValue( "e" ) );
111 
112 		if( cmd.hasOption( "s" ) )
113 			setTestSuite( getCommandLineOptionSubstSpace( cmd, "s" ) );
114 
115 		if( cmd.hasOption( "c" ) )
116 			setTestCase( getCommandLineOptionSubstSpace( cmd, "c" ) );
117 
118 		if( cmd.hasOption( "u" ) )
119 			setUsername( cmd.getOptionValue( "u" ) );
120 
121 		if( cmd.hasOption( "p" ) )
122 			setPassword( cmd.getOptionValue( "p" ) );
123 
124 		if( cmd.hasOption( "w" ) )
125 			setWssPasswordType( cmd.getOptionValue( "w" ) );
126 
127 		if( cmd.hasOption( "d" ) )
128 			setDomain( cmd.getOptionValue( "d" ) );
129 
130 		if( cmd.hasOption( "h" ) )
131 			setHost( cmd.getOptionValue( "h" ) );
132 
133 		if( cmd.hasOption( "f" ) )
134 			setOutputFolder( getCommandLineOptionSubstSpace( cmd, "f" ) );
135 
136 		if( cmd.hasOption( "t" ) )
137 			setSettingsFile( getCommandLineOptionSubstSpace( cmd, "t" ) );
138 
139 		if( cmd.hasOption( "x" ) )
140 		{
141 			setProjectPassword( cmd.getOptionValue( "x" ) );
142 		}
143 
144 		if( cmd.hasOption( "v" ) )
145 		{
146 			setSoapUISettingsPassword( cmd.getOptionValue( "v" ) );
147 		}
148 
149 		if( cmd.hasOption( "D" ) )
150 		{
151 			setSystemProperties( cmd.getOptionValues( "D" ) );
152 		}
153 
154 		if( cmd.hasOption( "G" ) )
155 		{
156 			setGlobalProperties( cmd.getOptionValues( "G" ) );
157 		}
158 
159 		if( cmd.hasOption( "P" ) )
160 		{
161 			setProjectProperties( cmd.getOptionValues( "P" ) );
162 		}
163 
164 		setIgnoreError( cmd.hasOption( "I" ) );
165 		setEnableUI( cmd.hasOption( "i" ) );
166 		setPrintReport( cmd.hasOption( "r" ) );
167 		setExportAll( cmd.hasOption( "a" ) );
168 		setJUnitReport( cmd.hasOption( "j" ) );
169 		setSaveAfterRun( cmd.hasOption( "S" ) );
170 
171 		return true;
172 	}
173 
174 	public void setSaveAfterRun( boolean saveAfterRun )
175 	{
176 		this.saveAfterRun = saveAfterRun;
177 	}
178 
179 	public void setProjectPassword( String projectPassword )
180 	{
181 		this.projectPassword = projectPassword;
182 	}
183 
184 	public String getProjectPassword()
185 	{
186 		return projectPassword;
187 	}
188 
189 	protected SoapUIOptions initCommandLineOptions()
190 	{
191 		SoapUIOptions options = new SoapUIOptions( "testrunner" );
192 		options.addOption( "e", true, "Sets the endpoint" );
193 		options.addOption( "s", true, "Sets the testsuite" );
194 		options.addOption( "c", true, "Sets the testcase" );
195 		options.addOption( "u", true, "Sets the username" );
196 		options.addOption( "p", true, "Sets the password" );
197 		options.addOption( "w", true, "Sets the WSS password type, either 'Text' or 'Digest'" );
198 		options.addOption( "i", false, "Enables Swing UI for scripts" );
199 		options.addOption( "d", true, "Sets the domain" );
200 		options.addOption( "h", true, "Sets the host" );
201 		options.addOption( "r", false, "Prints a small summary report" );
202 		options.addOption( "f", true, "Sets the output folder to export results to" );
203 		options.addOption( "j", false, "Sets the output to include JUnit XML reports" );
204 		options.addOption( "a", false, "Turns on exporting of all results" );
205 		options.addOption( "t", true, "Sets the soapui-settings.xml file to use" );
206 		options.addOption( "x", true, "Sets project password for decryption if project is encrypted" );
207 		options.addOption( "v", true, "Sets password for soapui-settings.xml file" );
208 		options.addOption( "D", true, "Sets system property with name=value" );
209 		options.addOption( "G", true, "Sets global property with name=value" );
210 		options.addOption( "P", true, "Sets or overrides project property with name=value" );
211 		options.addOption( "I", false, "Do not stop if error occurs, ignore them" );
212 		options.addOption( "S", false, "Saves the project after running the tests" );
213 
214 		return options;
215 	}
216 
217 	/***
218 	 * Add console appender to groovy log
219 	 */
220 
221 	public void setExportAll( boolean exportAll )
222 	{
223 		this.exportAll = exportAll;
224 	}
225 
226 	public void setJUnitReport( boolean junitReport )
227 	{
228 		this.junitReport = junitReport;
229 		if( junitReport )
230 			reportCollector = new JUnitReportCollector();
231 	}
232 
233 	public SoapUITestCaseRunner()
234 	{
235 		super( SoapUITestCaseRunner.TITLE );
236 	}
237 
238 	public SoapUITestCaseRunner( String title )
239 	{
240 		super( title );
241 	}
242 
243 	/***
244 	 * Controls if a short test summary should be printed after the test runs
245 	 * 
246 	 * @param printReport
247 	 *           a flag controlling if a summary should be printed
248 	 */
249 
250 	public void setPrintReport( boolean printReport )
251 	{
252 		this.printReport = printReport;
253 	}
254 
255 	public void setIgnoreError( boolean ignoreErrors )
256 	{
257 		this.ignoreErrors = ignoreErrors;
258 	}
259 
260 	public boolean runRunner() throws Exception
261 	{
262 		initGroovyLog();
263 
264 		assertions.clear();
265 
266 		String projectFile = getProjectFile();
267 
268 		WsdlProject project = ( WsdlProject )ProjectFactoryRegistry.getProjectFactory( "wsdl" ).createNew( projectFile,
269 				getProjectPassword() );
270 
271 		if( project.isDisabled() )
272 			throw new Exception( "Failed to load soapUI project file [" + projectFile + "]" );
273 
274 		initProject( project );
275 		ensureOutputFolder( project );
276 
277 		log.info( "Running soapUI tests in project [" + project.getName() + "]" );
278 
279 		long startTime = System.nanoTime();
280 
281 		List<TestCase> testCasesToRun = new ArrayList<TestCase>();
282 		
283 		// start by listening to all testcases.. (since one testcase can call
284 		// another)
285 		for( int c = 0; c < project.getTestSuiteCount(); c++ )
286 		{
287 			TestSuite suite = project.getTestSuiteAt( c );
288 			for( int i = 0; i < suite.getTestCaseCount(); i++ )
289 			{
290 				TestCase tc = suite.getTestCaseAt( i );
291 				if( testSuite == null && testCase != null && tc.getName().equals( testCase ))
292 					testCasesToRun.add( tc );
293 					
294 				addListeners( tc );
295 			}
296 		}
297 
298 		// decide what to run
299 		if( testCasesToRun.size() > 0 )
300 		{
301 			for( TestCase testCase : testCasesToRun )
302 			   runTestCase( ( WsdlTestCase )testCase );
303 		}
304 		else if( testSuite != null )
305 		{
306 			runSuite( project.getTestSuiteByName( testSuite ) );
307 		}
308 		else
309 		{
310 			runProject( project );
311 		}
312 
313 		long timeTaken = ( System.nanoTime() - startTime ) / 1000000;
314 
315 		if( printReport )
316 		{
317 			printReport( timeTaken );
318 		}
319 
320 		exportReports( project );
321 
322 		if( saveAfterRun && !project.isRemote() )
323 		{
324 			try
325 			{
326 				project.save();
327 			}
328 			catch( Throwable t )
329 			{
330 				log.error( "Failed to save project", t );
331 			}
332 		}
333 
334 		if( ( assertions.size() > 0 || failedTests.size() > 0 ) && !ignoreErrors )
335 		{
336 			throwFailureException();
337 		}
338 
339 		return true;
340 	}
341 
342 	protected void runProject( WsdlProject project )
343 	{
344 		try
345 		{
346 			log.info( ( "Running Project [" + project.getName() + "], runType = " + project.getRunType() ) );
347 			WsdlProjectRunner runner = project.run( new StringToObjectMap(), false );
348 			log.info( "Project [" + project.getName() + "] finished with status [" + runner.getStatus()
349 					+"] in " + runner.getTimeTaken() + "ms" );
350 		}
351 		catch( Exception e )
352 		{
353 			e.printStackTrace();
354 		}
355 	}
356 
357 	protected void initProject( WsdlProject project ) throws Exception
358 	{
359 		initProjectProperties( project );
360 	}
361 
362 	protected void exportReports( WsdlProject project ) throws Exception
363 	{
364 		if( junitReport )
365 		{
366 			exportJUnitReports( reportCollector, getAbsoluteOutputFolder( project ), project );
367 		}
368 	}
369 
370 	protected void addListeners( TestCase tc )
371 	{
372 		tc.addTestRunListener( this );
373 		if( junitReport )
374 			tc.addTestRunListener( reportCollector );
375 	}
376 
377 	protected void throwFailureException() throws Exception
378 	{
379 		StringBuffer buf = new StringBuffer();
380 
381 		for( int c = 0; c < assertions.size(); c++ )
382 		{
383 			TestAssertion assertion = assertions.get( c );
384 			Assertable assertable = assertion.getAssertable();
385 			if( assertable instanceof WsdlTestStep )
386 				failedTests.remove( ( ( WsdlTestStep )assertable ).getTestCase() );
387 
388 			buf.append( assertion.getName() + " in [" + assertable.getModelItem().getName() + "] failed;\n" );
389 			buf.append( Arrays.toString( assertion.getErrors() ) + "\n" );
390 
391 			WsdlTestStepResult result = assertionResults.get( assertion );
392 			StringWriter stringWriter = new StringWriter();
393 			PrintWriter writer = new PrintWriter( stringWriter );
394 			result.writeTo( writer );
395 			buf.append( stringWriter.toString() );
396 		}
397 
398 		while( !failedTests.isEmpty() )
399 		{
400 			buf.append( "TestCase [" + failedTests.remove( 0 ).getName() + "] failed without assertions\n" );
401 		}
402 
403 		throw new Exception( buf.toString() );
404 	}
405 
406 	public void exportJUnitReports( JUnitReportCollector collector, String folder, WsdlProject project ) throws Exception
407 	{
408 		collector.saveReports( folder == null ? "" : folder );
409 	}
410 
411 	public void printReport( long timeTaken )
412 	{
413 		System.out.println();
414 		System.out.println( "SoapUI " + SoapUI.SOAPUI_VERSION + " TestCaseRunner Summary" );
415 		System.out.println( "-----------------------------" );
416 		System.out.println( "Time Taken: " + timeTaken + "ms" );
417 		System.out.println( "Total TestSuites: " + testSuiteCount );
418 		System.out.println( "Total TestCases: " + testCaseCount + " (" + failedTests.size() + " failed)" );
419 		System.out.println( "Total TestSteps: " + testStepCount );
420 		System.out.println( "Total Request Assertions: " + testAssertionCount );
421 		System.out.println( "Total Failed Assertions: " + assertions.size() );
422 		System.out.println( "Total Exported Results: " + exportCount );
423 	}
424 
425 	/***
426 	 * Run tests in the specified TestSuite
427 	 * 
428 	 * @param suite
429 	 *           the TestSuite to run
430 	 */
431 
432 	protected void runSuite( WsdlTestSuite suite )
433 	{
434 		try
435 		{
436 			log.info( ( "Running TestSuite [" + suite.getName() + "], runType = " + suite.getRunType() ) );
437 		   WsdlTestSuiteRunner runner = suite.run( new StringToObjectMap(), false );
438 		   log.info( "TestSuite [" + suite.getName() + "] finished with status [" + runner.getStatus() + "] in " + ( runner.getTimeTaken() ) + "ms" );
439 		}
440 		catch( Exception e )
441 		{
442 			e.printStackTrace();
443 		}
444 	}
445 
446 	/***
447 	 * Runs the specified TestCase
448 	 * 
449 	 * @param testCase
450 	 *           the testcase to run
451 	 * @param context
452 	 */
453 
454 	protected void runTestCase( WsdlTestCase testCase )
455 	{
456 		try
457 		{
458 			log.info( "Running TestCase [" + testCase.getName() + "]" );
459 			WsdlTestCaseRunner runner = testCase.run( new StringToObjectMap(), false );
460 			log.info( "TestCase [" + testCase.getName() + "] finished with status [" + runner.getStatus() + "] in " + ( runner.getTimeTaken() ) + "ms" );
461 		}
462 		catch( Exception e )
463 		{
464 			e.printStackTrace();
465 		}
466 	}
467 
468 	/***
469 	 * Sets the testcase to run
470 	 * 
471 	 * @param testCase
472 	 *           the testcase to run
473 	 */
474 
475 	public void setTestCase( String testCase )
476 	{
477 		this.testCase = testCase;
478 	}
479 
480 	/***
481 	 * Sets the TestSuite to run. If not set all TestSuites in the specified
482 	 * project file are run
483 	 * 
484 	 * @param testSuite
485 	 *           the testSuite to run.
486 	 */
487 
488 	public void setTestSuite( String testSuite )
489 	{
490 		this.testSuite = testSuite;
491 	}
492 
493 	public void beforeRun( TestCaseRunner testRunner, TestCaseRunContext runContext )
494 	{
495 		log.info( "Running soapUI testcase [" + testRunner.getTestCase().getName() + "]" );
496 	}
497 
498 	public void beforeStep( TestCaseRunner testRunner, TestCaseRunContext runContext, TestStep currentStep )
499 	{
500 		super.beforeStep( testRunner, runContext, currentStep );
501 
502 		if( currentStep != null )
503 		   log.info( "running step [" + currentStep.getName() + "]" );
504 	}
505 
506 	public void afterStep( TestCaseRunner testRunner, TestCaseRunContext runContext, TestStepResult result )
507 	{
508 		super.afterStep( testRunner, runContext, result );
509 		TestStep currentStep = runContext.getCurrentStep();
510 
511 		if( currentStep instanceof Assertable )
512 		{
513 			Assertable requestStep = ( Assertable )currentStep;
514 			for( int c = 0; c < requestStep.getAssertionCount(); c++ )
515 			{
516 				TestAssertion assertion = requestStep.getAssertionAt( c );
517 				log.info( "Assertion [" + assertion.getName() + "] has status " + assertion.getStatus() );
518 				if( assertion.getStatus() == AssertionStatus.FAILED )
519 				{
520 					for( AssertionError error : assertion.getErrors() )
521 						log.error( "ASSERTION FAILED -> " + error.getMessage() );
522 
523 					assertions.add( assertion );
524 					assertionResults.put( assertion, ( WsdlTestStepResult )result );
525 				}
526 
527 				testAssertionCount++ ;
528 			}
529 		}
530 
531 		String countPropertyName = currentStep.getName() + " run count";
532 		Long count = ( Long )runContext.getProperty( countPropertyName );
533 		if( count == null )
534 		{
535 			count = new Long( 0 );
536 		}
537 
538 		runContext.setProperty( countPropertyName, new Long( count.longValue() + 1 ) );
539 
540 		if( result.getStatus() == TestStepStatus.FAILED || exportAll )
541 		{
542 			try
543 			{
544 				TestCase tc = currentStep.getTestCase();
545 				String nameBase = StringUtils.createFileName( tc.getTestSuite().getName(), '_' ) + "-"
546 						+ StringUtils.createFileName( tc.getName(), '_' ) + "-"
547 						+ StringUtils.createFileName( currentStep.getName(), '_' ) + "-" + count.longValue() + "-"
548 						+ result.getStatus();
549 
550 				String absoluteOutputFolder = getAbsoluteOutputFolder( ModelSupport.getModelItemProject( tc ));
551 				String fileName = absoluteOutputFolder + File.separator + nameBase + ".txt";
552 
553 				if( result.getStatus() == TestStepStatus.FAILED )
554 					log.error( currentStep.getName() + " failed, exporting to [" + fileName + "]" );
555 
556 				PrintWriter writer = new PrintWriter( fileName );
557 				result.writeTo( writer );
558 				writer.close();
559 
560 				// write attachments
561 				if( result instanceof WsdlTestRequestStepResult )
562 				{
563 					Attachment[] attachments = ( ( WsdlTestRequestStepResult )result ).getResponseAttachments();
564 					if( attachments != null && attachments.length > 0 )
565 					{
566 						for( int c = 0; c < attachments.length; c++ )
567 						{
568 							fileName = nameBase + "-attachment-" + ( c + 1 ) + ".";
569 
570 							Attachment attachment = attachments[c];
571 							String contentType = attachment.getContentType();
572 							if( !"application/octet-stream".equals( contentType ) && contentType != null
573 									&& contentType.indexOf( '/' ) != -1 )
574 							{
575 								fileName += contentType.substring( contentType.lastIndexOf( '/' ) + 1 );
576 							}
577 							else
578 							{
579 								fileName += "dat";
580 							}
581 
582 							fileName = absoluteOutputFolder + File.separator + fileName;
583 
584 							FileOutputStream outFile = new FileOutputStream( fileName );
585 							Tools.writeAll( outFile, attachment.getInputStream() );
586 							outFile.close();
587 						}
588 					}
589 				}
590 
591 				exportCount++ ;
592 			}
593 			catch( Exception e )
594 			{
595 				log.error( "Error saving failed result: " + e, e );
596 			}
597 		}
598 
599 		testStepCount++ ;
600 	}
601 
602 	public void afterRun( TestCaseRunner testRunner, TestCaseRunContext runContext )
603 	{
604 		log.info( "Finished running soapUI testcase [" + testRunner.getTestCase().getName() + "], time taken: "
605 				+ testRunner.getTimeTaken() + "ms, status: " + testRunner.getStatus() );
606 
607 		if( testRunner.getStatus() == Status.FAILED )
608 		{
609 			failedTests.add( testRunner.getTestCase() );
610 		}
611 
612 		testCaseCount++ ;
613 	}
614 }