View Javadoc

1   /*
2    *  soapUI, copyright (C) 2004-2009 eviware.com 
3    *
4    *  soapUI is free software; you can redistribute it and/or modify it under the 
5    *  terms of version 2.1 of the GNU Lesser General Public License as published by 
6    *  the Free Software Foundation.
7    *
8    *  soapUI is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without 
9    *  even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 
10   *  See the GNU Lesser General Public License for more details at gnu.org.
11   */
12  
13  package com.eviware.soapui.tools;
14  
15  import java.io.File;
16  import java.io.IOException;
17  import java.util.ArrayList;
18  import java.util.List;
19  
20  import org.apache.commons.cli.CommandLine;
21  
22  import com.eviware.soapui.SoapUI;
23  import com.eviware.soapui.impl.wsdl.WsdlProject;
24  import com.eviware.soapui.impl.wsdl.loadtest.WsdlLoadTest;
25  import com.eviware.soapui.impl.wsdl.loadtest.data.actions.ExportLoadTestLogAction;
26  import com.eviware.soapui.impl.wsdl.loadtest.data.actions.ExportStatisticsAction;
27  import com.eviware.soapui.impl.wsdl.loadtest.log.LoadTestLog;
28  import com.eviware.soapui.impl.wsdl.loadtest.log.LoadTestLogEntry;
29  import com.eviware.soapui.model.project.ProjectFactoryRegistry;
30  import com.eviware.soapui.model.testsuite.LoadTestRunContext;
31  import com.eviware.soapui.model.testsuite.LoadTestRunListener;
32  import com.eviware.soapui.model.testsuite.LoadTestRunner;
33  import com.eviware.soapui.model.testsuite.TestCase;
34  import com.eviware.soapui.model.testsuite.TestCaseRunContext;
35  import com.eviware.soapui.model.testsuite.TestCaseRunner;
36  import com.eviware.soapui.model.testsuite.TestStep;
37  import com.eviware.soapui.model.testsuite.TestStepResult;
38  import com.eviware.soapui.model.testsuite.TestSuite;
39  import com.eviware.soapui.settings.UISettings;
40  import com.eviware.soapui.support.SoapUIException;
41  import com.eviware.soapui.support.StringUtils;
42  
43  /***
44   * Standalone test-runner used from maven-plugin, can also be used from
45   * command-line (see xdocs) or directly from other classes.
46   * <p>
47   * For standalone usage, set the project file (with setProjectFile) and other
48   * desired properties before calling run
49   * </p>
50   * 
51   * @author Ole.Matzura
52   */
53  
54  public class SoapUILoadTestRunner extends AbstractSoapUITestRunner implements LoadTestRunListener
55  {
56  	private String testSuite;
57  	private String testCase;
58  	private String loadTest;
59  	private boolean printReport;
60  	private List<LoadTestRunner> failedTests = new ArrayList<LoadTestRunner>();
61  	private int testCaseCount;
62  	private int loadTestCount;
63  	private int limit = -1;
64  	private long threadCount = -1;
65  	private boolean saveAfterRun;
66  
67  	public static String TITLE = "soapUI " + SoapUI.SOAPUI_VERSION + " LoadTest Runner";
68  
69  	/***
70  	 * Runs the loadtests in the specified soapUI project file, see soapUI xdocs
71  	 * for details.
72  	 * 
73  	 * @param args
74  	 * @throws Exception
75  	 */
76  
77  	public static void main( String[] args )
78  	{
79  		System.exit( new SoapUILoadTestRunner().runFromCommandLine( args ));
80  	}
81  
82  	protected boolean processCommandLine( CommandLine cmd )
83  	{
84  		if( cmd.hasOption( "e" ) )
85  			setEndpoint( cmd.getOptionValue( "e" ) );
86  
87  		if( cmd.hasOption( "s" ) )
88  			setTestSuite( getCommandLineOptionSubstSpace( cmd, "s" ) );
89  
90  		if( cmd.hasOption( "c" ) )
91  			setTestCase( cmd.getOptionValue( "c" ) );
92  
93  		if( cmd.hasOption( "l" ) )
94  			setLoadTest( cmd.getOptionValue( "l" ) );
95  
96  		if( cmd.hasOption( "u" ) )
97  			setUsername( cmd.getOptionValue( "u" ) );
98  
99  		if( cmd.hasOption( "p" ) )
100 			setPassword( cmd.getOptionValue( "p" ) );
101 
102 		if( cmd.hasOption( "w" ) )
103 			setWssPasswordType( cmd.getOptionValue( "w" ) );
104 
105 		if( cmd.hasOption( "d" ) )
106 			setDomain( cmd.getOptionValue( "d" ) );
107 
108 		if( cmd.hasOption( "h" ) )
109 			setHost( cmd.getOptionValue( "h" ) );
110 
111 		if( cmd.hasOption( "m" ) )
112 			setLimit( Integer.parseInt( cmd.getOptionValue( "m" ) ) );
113 
114 		if( cmd.hasOption( "n" ) )
115 			setThreadCount( Integer.parseInt( cmd.getOptionValue( "n" ) ) );
116 
117 		if( cmd.hasOption( "f" ) )
118 			setOutputFolder( getCommandLineOptionSubstSpace( cmd, "f" ) );
119 
120 		if( cmd.hasOption( "t" ) )
121 			setSettingsFile( getCommandLineOptionSubstSpace( cmd, "t" ) );
122 
123 		setPrintReport( cmd.hasOption( "r" ) );
124 		setSaveAfterRun( cmd.hasOption( "S" ) );
125 
126 		if( cmd.hasOption( "x" ) )
127 		{
128 			setProjectPassword( cmd.getOptionValue( "x" ) );
129 		}
130 
131 		if( cmd.hasOption( "v" ) )
132 		{
133 			setSoapUISettingsPassword( cmd.getOptionValue( "v" ) );
134 		}
135 
136 		if( cmd.hasOption( "D" ) )
137 		{
138 			setSystemProperties( cmd.getOptionValues( "D" ) );
139 		}
140 
141 		if( cmd.hasOption( "G" ) )
142 		{
143 			setGlobalProperties( cmd.getOptionValues( "G" ) );
144 		}
145 
146 		if( cmd.hasOption( "P" ) )
147 		{
148 			setProjectProperties( cmd.getOptionValues( "P" ) );
149 		}
150 
151 		return true;
152 	}
153 
154 	public void setLimit( int limit )
155 	{
156 		this.limit = limit;
157 	}
158 
159 	public void setThreadCount( long threadCount )
160 	{
161 		this.threadCount = threadCount;
162 	}
163 
164 	protected SoapUIOptions initCommandLineOptions()
165 	{
166 		SoapUIOptions options = new SoapUIOptions( "loadtestrunner" );
167 		options.addOption( "e", true, "Sets the endpoint" );
168 		options.addOption( "s", true, "Sets the testsuite" );
169 		options.addOption( "c", true, "Sets the testcase" );
170 		options.addOption( "l", true, "Sets the loadtest" );
171 		options.addOption( "u", true, "Sets the username" );
172 		options.addOption( "p", true, "Sets the password" );
173 		options.addOption( "w", true, "Sets the WSS password type, either 'Text' or 'Digest'" );
174 		options.addOption( "d", true, "Sets the domain" );
175 		options.addOption( "h", true, "Sets the host" );
176 		options.addOption( "m", true, "Overrides the LoadTest Limit" );
177 		options.addOption( "n", true, "Overrides the LoadTest ThreadCount" );
178 		options.addOption( "r", false, "Exports statistics and testlogs for each LoadTest run" );
179 		options.addOption( "f", true, "Sets the output folder to export to" );
180 		options.addOption( "t", true, "Sets the soapui-settings.xml file to use" );
181 		options.addOption( "x", true, "Sets project password for decryption if project is encrypted" );
182 		options.addOption( "v", true, "Sets password for soapui-settings.xml file" );
183 		options.addOption( "D", true, "Sets system property with name=value" );
184 		options.addOption( "G", true, "Sets global property with name=value" );
185 		options.addOption( "P", true, "Sets or overrides project property with name=value" );
186 		options.addOption( "S", false, "Saves the project after running the tests" );
187 
188 		return options;
189 	}
190 
191 	public SoapUILoadTestRunner()
192 	{
193 		this( TITLE );
194 	}
195 
196 	public SoapUILoadTestRunner( String title )
197 	{
198 		super( title );
199 	}
200 
201 	public void setLoadTest( String loadTest )
202 	{
203 		this.loadTest = loadTest;
204 	}
205 
206 	public void setPrintReport( boolean printReport )
207 	{
208 		this.printReport = printReport;
209 	}
210 
211 	public void setSaveAfterRun( boolean saveAfterRun )
212 	{
213 		this.saveAfterRun = saveAfterRun;
214 	}
215 
216 	/***
217 	 * Runs the testcases as configured with setXXX methods
218 	 * 
219 	 * @throws Exception
220 	 *            thrown if any tests fail
221 	 */
222 
223 	public boolean runRunner() throws Exception
224 	{
225 		if( SoapUI.getSettings().getBoolean( UISettings.DONT_DISABLE_GROOVY_LOG ) )
226 		{
227 			initGroovyLog();
228 		}
229 
230 		String projectFile = getProjectFile();
231 
232 		// WsdlProject project = new WsdlProject( projectFile,
233 		// getProjectPassword() );
234 		WsdlProject project = ( WsdlProject )ProjectFactoryRegistry.getProjectFactory( "wsdl" ).createNew( projectFile,
235 				getProjectPassword() );
236 
237 		if( project.isDisabled() )
238 			throw new Exception( "Failed to load soapUI project file [" + projectFile + "]" );
239 
240 		initProjectProperties( project );
241 
242 		int suiteCount = 0;
243 
244 		for( int c = 0; c < project.getTestSuiteCount(); c++ )
245 		{
246 			if( testSuite == null || project.getTestSuiteAt( c ).getName().equalsIgnoreCase( testSuite ) )
247 			{
248 				runSuite( project.getTestSuiteAt( c ) );
249 				suiteCount++ ;
250 			}
251 		}
252 
253 		if( suiteCount == 0 )
254 		{
255 			log.warn( "No test-suites matched argument [" + testSuite + "]" );
256 		}
257 		else if( testCaseCount == 0 )
258 		{
259 			log.warn( "No test-cases matched argument [" + testCase + "]" );
260 		}
261 		else if( loadTestCount == 0 )
262 		{
263 			log.warn( "No load-tests matched argument [" + loadTest + "]" );
264 		}
265 		else
266 		{
267 			if( saveAfterRun && !project.isRemote() )
268 			{
269 				try
270 				{
271 					project.save();
272 				}
273 				catch( Throwable t )
274 				{
275 					log.error( "Failed to save project", t );
276 				}
277 			}
278 
279 			if( !failedTests.isEmpty() )
280 		{
281 			log.info( failedTests.size() + " load tests failed:" );
282 			for( LoadTestRunner loadTestRunner : failedTests )
283 			{
284 				log.info( loadTestRunner.getLoadTest().getName() + ": " + loadTestRunner.getReason() );
285 			}
286 
287 			throw new SoapUIException( "LoadTests failed" );
288 		}
289 		}
290 
291 		return true;
292 	}
293 
294 	/***
295 	 * Run tests in the specified TestSuite
296 	 * 
297 	 * @param suite
298 	 *           the TestSuite to run
299 	 */
300 
301 	public void runSuite( TestSuite suite )
302 	{
303 		long start = System.currentTimeMillis();
304 		for( int c = 0; c < suite.getTestCaseCount(); c++ )
305 		{
306 			String name = suite.getTestCaseAt( c ).getName();
307 			if( testCase == null || name.equalsIgnoreCase( testCase ) )
308 			{
309 				runTestCase( suite.getTestCaseAt( c ) );
310 				testCaseCount++ ;
311 			}
312 			else
313 				log.info( "Skipping testcase [" + name + "], filter is [" + testCase + "]" );
314 		}
315 		log.info( "soapUI suite [" + suite.getName() + "] finished in " + ( System.currentTimeMillis() - start ) + "ms" );
316 	}
317 
318 	/***
319 	 * Runs the specified TestCase
320 	 * 
321 	 * @param testCase
322 	 *           the testcase to run
323 	 */
324 
325 	private void runTestCase( TestCase testCase )
326 	{
327 		for( int c = 0; c < testCase.getLoadTestCount(); c++ )
328 		{
329 			String name = testCase.getLoadTestAt( c ).getName();
330 			if( loadTest == null || loadTest.equalsIgnoreCase( name ) )
331 			{
332 				runWsdlLoadTest( ( WsdlLoadTest )testCase.getLoadTestAt( c ) );
333 				loadTestCount++ ;
334 			}
335 		}
336 	}
337 
338 	/***
339 	 * Runs the specified LoadTest
340 	 * 
341 	 * @param loadTest
342 	 *           the loadTest to run
343 	 */
344 
345 	protected void runWsdlLoadTest( WsdlLoadTest loadTest )
346 	{
347 		try
348 		{
349 			log.info( "Running LoadTest [" + loadTest.getName() + "]" );
350 			if( limit >= 0 )
351 			{
352 				log.info( "Overriding limit [" + loadTest.getTestLimit() + "] with specified [" + limit + "]" );
353 				loadTest.setTestLimit( limit );
354 			}
355 
356 			if( threadCount >= 0 )
357 			{
358 				log
359 						.info( "Overriding threadCount [" + loadTest.getThreadCount() + "] with specified [" + threadCount
360 								+ "]" );
361 				loadTest.setThreadCount( threadCount );
362 			}
363 
364 			loadTest.addLoadTestRunListener( this );
365 			LoadTestRunner runner = loadTest.run();
366 
367 			// wait for test to finish
368 			while( runner.getStatus() == LoadTestRunner.Status.RUNNING )
369 			{
370 				log.info( "LoadTest [" + loadTest.getName() + "] progress: " + runner.getProgress() + ", "
371 						+ runner.getRunningThreadCount() );
372 				Thread.sleep( 1000 );
373 			}
374 
375 			log.info( "LoadTest [" + loadTest.getName() + "] finished with status " + runner.getStatus().toString() );
376 
377 			if( printReport )
378 			{
379 				log.info( "Exporting log and statistics for LoadTest [" + loadTest.getName() + "]" );
380 
381 				loadTest.getStatisticsModel().finish();
382 
383 				exportLog( loadTest );
384 				exportStatistics( loadTest );
385 			}
386 		}
387 		catch( Exception e )
388 		{
389 			SoapUI.logError( e );
390 			log.error( e );
391 		}
392 	}
393 
394 	private void exportStatistics( WsdlLoadTest loadTest ) throws IOException
395 	{
396 		ExportStatisticsAction exportStatisticsAction = new ExportStatisticsAction( loadTest.getStatisticsModel() );
397 		String statisticsFileName = StringUtils.createFileName( loadTest.getName(), '_' ) + "-statistics.txt";
398 		if( getOutputFolder() != null )
399 		{
400 			ensureOutputFolder( loadTest );
401 			statisticsFileName = getAbsoluteOutputFolder( loadTest ) + File.separator + statisticsFileName;
402 		}
403 
404 		int cnt = exportStatisticsAction.exportToFile( new File( statisticsFileName ) );
405 		log.info( "Exported " + cnt + " statistics to [" + statisticsFileName + "]" );
406 	}
407 
408 	private void exportLog( WsdlLoadTest loadTest ) throws IOException
409 	{
410 		// export log first
411 		LoadTestLog loadTestLog = loadTest.getLoadTestLog();
412 		ExportLoadTestLogAction exportLoadTestLogAction = new ExportLoadTestLogAction( loadTestLog, null );
413 		String logFileName = StringUtils.createFileName( loadTest.getName(), '_' ) + "-log.txt";
414 		if( getOutputFolder() != null )
415 		{
416 			ensureOutputFolder( loadTest );
417 			logFileName = getAbsoluteOutputFolder( loadTest ) + File.separator + logFileName;
418 		}
419 
420 		int cnt = exportLoadTestLogAction.exportToFile( new File( logFileName ) );
421 		log.info( "Exported " + cnt + " log items to [" + logFileName + "]" );
422 
423 		int errorCnt = 0;
424 		for( int c = 0; c < loadTestLog.getSize(); c++ )
425 		{
426 			LoadTestLogEntry entry = ( LoadTestLogEntry )loadTestLog.getElementAt( c );
427 
428 			if( entry != null && entry.isError() )
429 			{
430 				String entryFileName = StringUtils.createFileName( loadTest.getName(), '_' ) + "-error-" + errorCnt++
431 						+ "-entry.txt";
432 				if( getOutputFolder() != null )
433 				{
434 					ensureOutputFolder( loadTest );
435 					entryFileName = getAbsoluteOutputFolder( loadTest ) + File.separator + entryFileName;
436 				}
437 
438 				try
439 				{
440 					entry.exportToFile( entryFileName );
441 				}
442 				catch( Exception e )
443 				{
444 					SoapUI.logError( e );
445 				}
446 			}
447 		}
448 		log.info( "Exported " + errorCnt + " error results" );
449 	}
450 
451 	/***
452 	 * Sets the testcase to run
453 	 * 
454 	 * @param testCase
455 	 *           the testcase to run
456 	 */
457 
458 	public void setTestCase( String testCase )
459 	{
460 		this.testCase = testCase;
461 	}
462 
463 	/***
464 	 * Sets the TestSuite to run. If not set all TestSuites in the specified
465 	 * project file are run
466 	 * 
467 	 * @param testSuite
468 	 *           the testSuite to run.
469 	 */
470 
471 	public void setTestSuite( String testSuite )
472 	{
473 		this.testSuite = testSuite;
474 	}
475 
476 	public void afterLoadTest( LoadTestRunner loadTestRunner, LoadTestRunContext context )
477 	{
478 		if( loadTestRunner.getStatus() == LoadTestRunner.Status.FAILED )
479 		{
480 			failedTests.add( loadTestRunner );
481 		}
482 	}
483 
484 	public void afterTestCase( LoadTestRunner loadTestRunner, LoadTestRunContext context, TestCaseRunner testRunner,
485 			TestCaseRunContext runContext )
486 	{
487 	}
488 
489 	public void afterTestStep( LoadTestRunner loadTestRunner, LoadTestRunContext context, TestCaseRunner testRunner,
490 			TestCaseRunContext runContext, TestStepResult testStepResult )
491 	{
492 		super.afterStep( testRunner, runContext, testStepResult );
493 	}
494 
495 	public void beforeLoadTest( LoadTestRunner loadTestRunner, LoadTestRunContext context )
496 	{
497 	}
498 
499 	public void beforeTestCase( LoadTestRunner loadTestRunner, LoadTestRunContext context, TestCaseRunner testRunner,
500 			TestCaseRunContext runContext )
501 	{
502 	}
503 
504 	public void beforeTestStep( LoadTestRunner loadTestRunner, LoadTestRunContext context, TestCaseRunner testRunner,
505 			TestCaseRunContext runContext, TestStep testStep )
506 	{
507 		super.beforeStep( testRunner, runContext, testStep );
508 	}
509 
510 	public void loadTestStarted( LoadTestRunner loadTestRunner, LoadTestRunContext context )
511 	{
512 	}
513 
514 	public void loadTestStopped( LoadTestRunner loadTestRunner, LoadTestRunContext context )
515 	{
516 	}
517 }