View Javadoc

1   /*
2    *  soapUI, copyright (C) 2004-2010 eviware.com 
3    *
4    *  soapUI is free software; you can redistribute it and/or modify it under the 
5    *  terms of version 2.1 of the GNU Lesser General Public License as published by 
6    *  the Free Software Foundation.
7    *
8    *  soapUI is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without 
9    *  even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 
10   *  See the GNU Lesser General Public License for more details at gnu.org.
11   */
12  
13  package com.eviware.soapui.tools;
14  
15  import java.io.File;
16  import java.io.IOException;
17  import java.util.ArrayList;
18  import java.util.List;
19  
20  import org.apache.commons.cli.CommandLine;
21  
22  import com.eviware.soapui.SoapUI;
23  import com.eviware.soapui.impl.wsdl.WsdlProject;
24  import com.eviware.soapui.impl.wsdl.loadtest.WsdlLoadTest;
25  import com.eviware.soapui.impl.wsdl.loadtest.data.actions.ExportLoadTestLogAction;
26  import com.eviware.soapui.impl.wsdl.loadtest.data.actions.ExportStatisticsAction;
27  import com.eviware.soapui.impl.wsdl.loadtest.log.LoadTestLog;
28  import com.eviware.soapui.impl.wsdl.loadtest.log.LoadTestLogEntry;
29  import com.eviware.soapui.model.project.ProjectFactoryRegistry;
30  import com.eviware.soapui.model.testsuite.LoadTestRunContext;
31  import com.eviware.soapui.model.testsuite.LoadTestRunListener;
32  import com.eviware.soapui.model.testsuite.LoadTestRunner;
33  import com.eviware.soapui.model.testsuite.TestCase;
34  import com.eviware.soapui.model.testsuite.TestCaseRunContext;
35  import com.eviware.soapui.model.testsuite.TestCaseRunner;
36  import com.eviware.soapui.model.testsuite.TestStep;
37  import com.eviware.soapui.model.testsuite.TestStepResult;
38  import com.eviware.soapui.model.testsuite.TestSuite;
39  import com.eviware.soapui.model.testsuite.TestRunner.Status;
40  import com.eviware.soapui.settings.UISettings;
41  import com.eviware.soapui.support.SoapUIException;
42  import com.eviware.soapui.support.StringUtils;
43  
44  /***
45   * Standalone test-runner used from maven-plugin, can also be used from
46   * command-line (see xdocs) or directly from other classes.
47   * <p>
48   * For standalone usage, set the project file (with setProjectFile) and other
49   * desired properties before calling run
50   * </p>
51   * 
52   * @author Ole.Matzura
53   */
54  
55  public class SoapUILoadTestRunner extends AbstractSoapUITestRunner implements LoadTestRunListener
56  {
57  	private String testSuite;
58  	private String testCase;
59  	private String loadTest;
60  	private boolean printReport;
61  	private List<LoadTestRunner> failedTests = new ArrayList<LoadTestRunner>();
62  	private int testCaseCount;
63  	private int loadTestCount;
64  	private int limit = -1;
65  	private long threadCount = -1;
66  	private boolean saveAfterRun;
67  
68  	public static String TITLE = "soapUI " + SoapUI.SOAPUI_VERSION + " LoadTest Runner";
69  
70  	/***
71  	 * Runs the loadtests in the specified soapUI project file, see soapUI xdocs
72  	 * for details.
73  	 * 
74  	 * @param args
75  	 * @throws Exception
76  	 */
77  
78  	public static void main( String[] args )
79  	{
80  		System.exit( new SoapUILoadTestRunner().runFromCommandLine( args ));
81  	}
82  
83  	protected boolean processCommandLine( CommandLine cmd )
84  	{
85  		if( cmd.hasOption( "e" ) )
86  			setEndpoint( cmd.getOptionValue( "e" ) );
87  
88  		if( cmd.hasOption( "s" ) )
89  			setTestSuite( getCommandLineOptionSubstSpace( cmd, "s" ) );
90  
91  		if( cmd.hasOption( "c" ) )
92  			setTestCase( cmd.getOptionValue( "c" ) );
93  
94  		if( cmd.hasOption( "l" ) )
95  			setLoadTest( cmd.getOptionValue( "l" ) );
96  
97  		if( cmd.hasOption( "u" ) )
98  			setUsername( cmd.getOptionValue( "u" ) );
99  
100 		if( cmd.hasOption( "p" ) )
101 			setPassword( cmd.getOptionValue( "p" ) );
102 
103 		if( cmd.hasOption( "w" ) )
104 			setWssPasswordType( cmd.getOptionValue( "w" ) );
105 
106 		if( cmd.hasOption( "d" ) )
107 			setDomain( cmd.getOptionValue( "d" ) );
108 
109 		if( cmd.hasOption( "h" ) )
110 			setHost( cmd.getOptionValue( "h" ) );
111 
112 		if( cmd.hasOption( "m" ) )
113 			setLimit( Integer.parseInt( cmd.getOptionValue( "m" ) ) );
114 
115 		if( cmd.hasOption( "n" ) )
116 			setThreadCount( Integer.parseInt( cmd.getOptionValue( "n" ) ) );
117 
118 		if( cmd.hasOption( "f" ) )
119 			setOutputFolder( getCommandLineOptionSubstSpace( cmd, "f" ) );
120 
121 		if( cmd.hasOption( "t" ) )
122 			setSettingsFile( getCommandLineOptionSubstSpace( cmd, "t" ) );
123 
124 		setPrintReport( cmd.hasOption( "r" ) );
125 		setSaveAfterRun( cmd.hasOption( "S" ) );
126 
127 		if( cmd.hasOption( "x" ) )
128 		{
129 			setProjectPassword( cmd.getOptionValue( "x" ) );
130 		}
131 
132 		if( cmd.hasOption( "v" ) )
133 		{
134 			setSoapUISettingsPassword( cmd.getOptionValue( "v" ) );
135 		}
136 
137 		if( cmd.hasOption( "D" ) )
138 		{
139 			setSystemProperties( cmd.getOptionValues( "D" ) );
140 		}
141 
142 		if( cmd.hasOption( "G" ) )
143 		{
144 			setGlobalProperties( cmd.getOptionValues( "G" ) );
145 		}
146 
147 		if( cmd.hasOption( "P" ) )
148 		{
149 			setProjectProperties( cmd.getOptionValues( "P" ) );
150 		}
151 
152 		return true;
153 	}
154 
155 	public void setLimit( int limit )
156 	{
157 		this.limit = limit;
158 	}
159 
160 	public void setThreadCount( long threadCount )
161 	{
162 		this.threadCount = threadCount;
163 	}
164 
165 	protected SoapUIOptions initCommandLineOptions()
166 	{
167 		SoapUIOptions options = new SoapUIOptions( "loadtestrunner" );
168 		options.addOption( "e", true, "Sets the endpoint" );
169 		options.addOption( "s", true, "Sets the testsuite" );
170 		options.addOption( "c", true, "Sets the testcase" );
171 		options.addOption( "l", true, "Sets the loadtest" );
172 		options.addOption( "u", true, "Sets the username" );
173 		options.addOption( "p", true, "Sets the password" );
174 		options.addOption( "w", true, "Sets the WSS password type, either 'Text' or 'Digest'" );
175 		options.addOption( "d", true, "Sets the domain" );
176 		options.addOption( "h", true, "Sets the host" );
177 		options.addOption( "m", true, "Overrides the LoadTest Limit" );
178 		options.addOption( "n", true, "Overrides the LoadTest ThreadCount" );
179 		options.addOption( "r", false, "Exports statistics and testlogs for each LoadTest run" );
180 		options.addOption( "f", true, "Sets the output folder to export to" );
181 		options.addOption( "t", true, "Sets the soapui-settings.xml file to use" );
182 		options.addOption( "x", true, "Sets project password for decryption if project is encrypted" );
183 		options.addOption( "v", true, "Sets password for soapui-settings.xml file" );
184 		options.addOption( "D", true, "Sets system property with name=value" );
185 		options.addOption( "G", true, "Sets global property with name=value" );
186 		options.addOption( "P", true, "Sets or overrides project property with name=value" );
187 		options.addOption( "S", false, "Saves the project after running the tests" );
188 
189 		return options;
190 	}
191 
192 	public SoapUILoadTestRunner()
193 	{
194 		this( TITLE );
195 	}
196 
197 	public SoapUILoadTestRunner( String title )
198 	{
199 		super( title );
200 	}
201 
202 	public void setLoadTest( String loadTest )
203 	{
204 		this.loadTest = loadTest;
205 	}
206 
207 	public void setPrintReport( boolean printReport )
208 	{
209 		this.printReport = printReport;
210 	}
211 
212 	public void setSaveAfterRun( boolean saveAfterRun )
213 	{
214 		this.saveAfterRun = saveAfterRun;
215 	}
216 
217 	/***
218 	 * Runs the testcases as configured with setXXX methods
219 	 * 
220 	 * @throws Exception
221 	 *            thrown if any tests fail
222 	 */
223 
224 	public boolean runRunner() throws Exception
225 	{
226 		if( SoapUI.getSettings().getBoolean( UISettings.DONT_DISABLE_GROOVY_LOG ) )
227 		{
228 			initGroovyLog();
229 		}
230 
231 		String projectFile = getProjectFile();
232 
233 		// WsdlProject project = new WsdlProject( projectFile,
234 		// getProjectPassword() );
235 		WsdlProject project = ( WsdlProject )ProjectFactoryRegistry.getProjectFactory( "wsdl" ).createNew( projectFile,
236 				getProjectPassword() );
237 
238 		if( project.isDisabled() )
239 			throw new Exception( "Failed to load soapUI project file [" + projectFile + "]" );
240 
241 		initProjectProperties( project );
242 
243 		int suiteCount = 0;
244 
245 		for( int c = 0; c < project.getTestSuiteCount(); c++ )
246 		{
247 			if( testSuite == null || project.getTestSuiteAt( c ).getName().equalsIgnoreCase( testSuite ) )
248 			{
249 				runSuite( project.getTestSuiteAt( c ) );
250 				suiteCount++ ;
251 			}
252 		}
253 
254 		if( suiteCount == 0 )
255 		{
256 			log.warn( "No test-suites matched argument [" + testSuite + "]" );
257 		}
258 		else if( testCaseCount == 0 )
259 		{
260 			log.warn( "No test-cases matched argument [" + testCase + "]" );
261 		}
262 		else if( loadTestCount == 0 )
263 		{
264 			log.warn( "No load-tests matched argument [" + loadTest + "]" );
265 		}
266 		else
267 		{
268 			if( saveAfterRun && !project.isRemote() )
269 			{
270 				try
271 				{
272 					project.save();
273 				}
274 				catch( Throwable t )
275 				{
276 					log.error( "Failed to save project", t );
277 				}
278 			}
279 
280 			if( !failedTests.isEmpty() )
281 		{
282 			log.info( failedTests.size() + " load tests failed:" );
283 			for( LoadTestRunner loadTestRunner : failedTests )
284 			{
285 				log.info( loadTestRunner.getLoadTest().getName() + ": " + loadTestRunner.getReason() );
286 			}
287 
288 			throw new SoapUIException( "LoadTests failed" );
289 		}
290 		}
291 
292 		return true;
293 	}
294 
295 	/***
296 	 * Run tests in the specified TestSuite
297 	 * 
298 	 * @param suite
299 	 *           the TestSuite to run
300 	 */
301 
302 	public void runSuite( TestSuite suite )
303 	{
304 		long start = System.currentTimeMillis();
305 		for( int c = 0; c < suite.getTestCaseCount(); c++ )
306 		{
307 			String name = suite.getTestCaseAt( c ).getName();
308 			if( testCase == null || name.equalsIgnoreCase( testCase ) )
309 			{
310 				runTestCase( suite.getTestCaseAt( c ) );
311 				testCaseCount++ ;
312 			}
313 			else
314 				log.info( "Skipping testcase [" + name + "], filter is [" + testCase + "]" );
315 		}
316 		log.info( "soapUI suite [" + suite.getName() + "] finished in " + ( System.currentTimeMillis() - start ) + "ms" );
317 	}
318 
319 	/***
320 	 * Runs the specified TestCase
321 	 * 
322 	 * @param testCase
323 	 *           the testcase to run
324 	 */
325 
326 	private void runTestCase( TestCase testCase )
327 	{
328 		for( int c = 0; c < testCase.getLoadTestCount(); c++ )
329 		{
330 			String name = testCase.getLoadTestAt( c ).getName();
331 			if( loadTest == null || loadTest.equalsIgnoreCase( name ) )
332 			{
333 				runWsdlLoadTest( ( WsdlLoadTest )testCase.getLoadTestAt( c ) );
334 				loadTestCount++ ;
335 			}
336 		}
337 	}
338 
339 	/***
340 	 * Runs the specified LoadTest
341 	 * 
342 	 * @param loadTest
343 	 *           the loadTest to run
344 	 */
345 
346 	protected void runWsdlLoadTest( WsdlLoadTest loadTest )
347 	{
348 		try
349 		{
350 			log.info( "Running LoadTest [" + loadTest.getName() + "]" );
351 			if( limit >= 0 )
352 			{
353 				log.info( "Overriding limit [" + loadTest.getTestLimit() + "] with specified [" + limit + "]" );
354 				loadTest.setTestLimit( limit );
355 			}
356 
357 			if( threadCount >= 0 )
358 			{
359 				log.info( "Overriding threadCount [" + loadTest.getThreadCount() + "] with specified [" + threadCount
360 								+ "]" );
361 				loadTest.setThreadCount( threadCount );
362 			}
363 
364 			loadTest.addLoadTestRunListener( this );
365 			LoadTestRunner runner = loadTest.run();
366 
367 			// wait for test to finish
368 			while( !runner.hasStopped() )
369 			{
370 				if( runner.getStatus() == Status.RUNNING )
371 			{
372 				log.info( "LoadTest [" + loadTest.getName() + "] progress: " + runner.getProgress() + ", "
373 						+ runner.getRunningThreadCount() );
374 				}
375 				Thread.sleep( 1000 );
376 			}
377 
378 			log.info( "LoadTest [" + loadTest.getName() + "] finished with status " + runner.getStatus().toString() );
379 
380 			if( printReport )
381 			{
382 				log.info( "Exporting log and statistics for LoadTest [" + loadTest.getName() + "]" );
383 
384 				loadTest.getStatisticsModel().finish();
385 
386 				exportLog( loadTest );
387 				exportStatistics( loadTest );
388 			}
389 		}
390 		catch( Exception e )
391 		{
392 			SoapUI.logError( e );
393 			log.error( e );
394 		}
395 	}
396 
397 	private void exportStatistics( WsdlLoadTest loadTest ) throws IOException
398 	{
399 		ExportStatisticsAction exportStatisticsAction = new ExportStatisticsAction( loadTest.getStatisticsModel() );
400 		String statisticsFileName = StringUtils.createFileName( loadTest.getName(), '_' ) + "-statistics.txt";
401 		if( getOutputFolder() != null )
402 		{
403 			ensureOutputFolder( loadTest );
404 			statisticsFileName = getAbsoluteOutputFolder( loadTest ) + File.separator + statisticsFileName;
405 		}
406 
407 		int cnt = exportStatisticsAction.exportToFile( new File( statisticsFileName ) );
408 		log.info( "Exported " + cnt + " statistics to [" + statisticsFileName + "]" );
409 	}
410 
411 	private void exportLog( WsdlLoadTest loadTest ) throws IOException
412 	{
413 		// export log first
414 		LoadTestLog loadTestLog = loadTest.getLoadTestLog();
415 		ExportLoadTestLogAction exportLoadTestLogAction = new ExportLoadTestLogAction( loadTestLog, null );
416 		String logFileName = StringUtils.createFileName( loadTest.getName(), '_' ) + "-log.txt";
417 		if( getOutputFolder() != null )
418 		{
419 			ensureOutputFolder( loadTest );
420 			logFileName = getAbsoluteOutputFolder( loadTest ) + File.separator + logFileName;
421 		}
422 
423 		int cnt = exportLoadTestLogAction.exportToFile( new File( logFileName ) );
424 		log.info( "Exported " + cnt + " log items to [" + logFileName + "]" );
425 
426 		int errorCnt = 0;
427 		for( int c = 0; c < loadTestLog.getSize(); c++ )
428 		{
429 			LoadTestLogEntry entry = ( LoadTestLogEntry )loadTestLog.getElementAt( c );
430 
431 			if( entry != null && entry.isError() )
432 			{
433 				String entryFileName = StringUtils.createFileName( loadTest.getName(), '_' ) + "-error-" + errorCnt++
434 						+ "-entry.txt";
435 				if( getOutputFolder() != null )
436 				{
437 					ensureOutputFolder( loadTest );
438 					entryFileName = getAbsoluteOutputFolder( loadTest ) + File.separator + entryFileName;
439 				}
440 
441 				try
442 				{
443 					entry.exportToFile( entryFileName );
444 				}
445 				catch( Exception e )
446 				{
447 					SoapUI.logError( e );
448 				}
449 			}
450 		}
451 		log.info( "Exported " + errorCnt + " error results" );
452 	}
453 
454 	/***
455 	 * Sets the testcase to run
456 	 * 
457 	 * @param testCase
458 	 *           the testcase to run
459 	 */
460 
461 	public void setTestCase( String testCase )
462 	{
463 		this.testCase = testCase;
464 	}
465 
466 	/***
467 	 * Sets the TestSuite to run. If not set all TestSuites in the specified
468 	 * project file are run
469 	 * 
470 	 * @param testSuite
471 	 *           the testSuite to run.
472 	 */
473 
474 	public void setTestSuite( String testSuite )
475 	{
476 		this.testSuite = testSuite;
477 	}
478 
479 	public void afterLoadTest( LoadTestRunner loadTestRunner, LoadTestRunContext context )
480 	{
481 		if( loadTestRunner.getStatus() == LoadTestRunner.Status.FAILED )
482 		{
483 			failedTests.add( loadTestRunner );
484 		}
485 	}
486 
487 	public void afterTestCase( LoadTestRunner loadTestRunner, LoadTestRunContext context, TestCaseRunner testRunner,
488 			TestCaseRunContext runContext )
489 	{
490 	}
491 
492 	public void afterTestStep( LoadTestRunner loadTestRunner, LoadTestRunContext context, TestCaseRunner testRunner,
493 			TestCaseRunContext runContext, TestStepResult testStepResult )
494 	{
495 		super.afterStep( testRunner, runContext, testStepResult );
496 	}
497 
498 	public void beforeLoadTest( LoadTestRunner loadTestRunner, LoadTestRunContext context )
499 	{
500 	}
501 
502 	public void beforeTestCase( LoadTestRunner loadTestRunner, LoadTestRunContext context, TestCaseRunner testRunner,
503 			TestCaseRunContext runContext )
504 	{
505 	}
506 
507 	public void beforeTestStep( LoadTestRunner loadTestRunner, LoadTestRunContext context, TestCaseRunner testRunner,
508 			TestCaseRunContext runContext, TestStep testStep )
509 	{
510 		super.beforeStep( testRunner, runContext, testStep );
511 	}
512 
513 	public void loadTestStarted( LoadTestRunner loadTestRunner, LoadTestRunContext context )
514 	{
515 	}
516 
517 	public void loadTestStopped( LoadTestRunner loadTestRunner, LoadTestRunContext context )
518 	{
519 	}
520 }