View Javadoc

1   /*
2    *  soapUI, copyright (C) 2004-2007 eviware.com 
3    *
4    *  soapUI is free software; you can redistribute it and/or modify it under the 
5    *  terms of version 2.1 of the GNU Lesser General Public License as published by 
6    *  the Free Software Foundation.
7    *
8    *  soapUI is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without 
9    *  even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 
10   *  See the GNU Lesser General Public License for more details at gnu.org.
11   */
12  
13  package com.eviware.soapui.tools;
14  
15  import java.io.File;
16  import java.io.IOException;
17  import java.util.ArrayList;
18  import java.util.List;
19  
20  import org.apache.commons.cli.CommandLine;
21  
22  import com.eviware.soapui.SoapUI;
23  import com.eviware.soapui.impl.wsdl.WsdlProject;
24  import com.eviware.soapui.impl.wsdl.loadtest.WsdlLoadTest;
25  import com.eviware.soapui.impl.wsdl.loadtest.data.actions.ExportLoadTestLogAction;
26  import com.eviware.soapui.impl.wsdl.loadtest.data.actions.ExportStatisticsAction;
27  import com.eviware.soapui.impl.wsdl.loadtest.log.LoadTestLog;
28  import com.eviware.soapui.impl.wsdl.loadtest.log.LoadTestLogEntry;
29  import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestRequest;
30  import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestRequestStep;
31  import com.eviware.soapui.model.testsuite.LoadTestRunContext;
32  import com.eviware.soapui.model.testsuite.LoadTestRunListener;
33  import com.eviware.soapui.model.testsuite.LoadTestRunner;
34  import com.eviware.soapui.model.testsuite.TestCase;
35  import com.eviware.soapui.model.testsuite.TestRunContext;
36  import com.eviware.soapui.model.testsuite.TestRunner;
37  import com.eviware.soapui.model.testsuite.TestStep;
38  import com.eviware.soapui.model.testsuite.TestStepResult;
39  import com.eviware.soapui.model.testsuite.TestSuite;
40  import com.eviware.soapui.settings.UISettings;
41  import com.eviware.soapui.support.SoapUIException;
42  import com.eviware.soapui.support.Tools;
43  
44  /***
45   * Standalone test-runner used from maven-plugin, can also be used from command-line (see xdocs) or
46   * directly from other classes.
47   * <p>
48   * For standalone usage, set the project file (with setProjectFile) and other desired properties before
49   * calling run</p> 
50   * 
51   * @author Ole.Matzura
52   */
53  
54  public class SoapUILoadTestRunner extends AbstractSoapUIRunner implements LoadTestRunListener
55  {
56  	private String testSuite;
57  	private String testCase;
58  	private String endpoint;
59  	private String domain;
60  	private String password;
61  	private String username;
62  	private String host;
63  	private String loadTest;
64  	private boolean printReport;
65  	private String outputFolder;
66  	private List<LoadTestRunner> failedTests = new ArrayList<LoadTestRunner>();
67  	private int testCaseCount;
68  	private int loadTestCount;
69  	private int limit = -1;
70  	private String wssPasswordType;
71  	
72  	public static String TITLE = "soapUI " + SoapUI.SOAPUI_VERSION + " LoadTest Runner";
73  	
74  	/***
75  	 * Runs the loadtests in the specified soapUI project file, see soapUI xdocs for details.
76  	 * 
77  	 * @param args
78  	 * @throws Exception
79  	 */
80  
81  	public static void main( String [] args ) 
82  	{
83  		new SoapUILoadTestRunner().runFromCommandLine( args );
84  	}
85  
86  	protected boolean processCommandLine( CommandLine cmd )
87  	{
88  		if( cmd.hasOption( "e"))
89  			setEndpoint( cmd.getOptionValue( "e" ) );
90  		
91  		if( cmd.hasOption( "s"))
92  			setTestSuite( cmd.getOptionValue( "s") );
93  
94  		if( cmd.hasOption( "c"))
95  			setTestCase( cmd.getOptionValue( "c") );
96  
97  		if( cmd.hasOption( "l"))
98  			setLoadTest( cmd.getOptionValue( "l") );
99  
100 		if( cmd.hasOption( "u"))
101 			setUsername( cmd.getOptionValue( "u") );
102 
103 		if( cmd.hasOption( "p"))
104 			setPassword( cmd.getOptionValue( "p") );
105 
106 		if( cmd.hasOption( "w"))
107 			setWssPasswordType( cmd.getOptionValue( "w") );
108 		
109 		if( cmd.hasOption( "d"))
110 			setDomain( cmd.getOptionValue( "d") );
111 
112 		if( cmd.hasOption( "h"))
113 			setHost( cmd.getOptionValue( "h") );
114 		
115 		if( cmd.hasOption( "m"))
116 			setLimit( Integer.parseInt( cmd.getOptionValue( "m") ));
117 		
118 		if( cmd.hasOption( "f"))
119 			setOutputFolder( cmd.getOptionValue( "f") );
120 		
121 		if( cmd.hasOption( "t"))
122 			setSettingsFile( cmd.getOptionValue( "t" ));
123 		
124 		setPrintReport( cmd.hasOption( "r") );
125 		
126 		return true;
127 	}
128 
129 	public void setLimit( int limit )
130 	{
131 		this.limit = limit;
132 	}
133 
134 	protected SoapUIOptions initCommandLineOptions()
135 	{
136 		SoapUIOptions options = new SoapUIOptions( "loadtestrunner" );
137 		options.addOption( "e", true, "Sets the endpoint" );
138 		options.addOption( "s", true, "Sets the testsuite" );
139 		options.addOption( "c", true, "Sets the testcase" );
140 		options.addOption( "l", true, "Sets the loadtest" );
141 		options.addOption( "u", true, "Sets the username" );
142 		options.addOption( "p", true, "Sets the password" );
143 		options.addOption( "w", true, "Sets the WSS password type, either 'Text' or 'Digest'" );
144 		options.addOption( "d", true, "Sets the domain" );
145 		options.addOption( "h", true, "Sets the host" );
146 		options.addOption( "m", true, "Overrides the LoadTest limit" );
147 		options.addOption( "r", false, "Exports statistics and testlogs for each LoadTest run" );
148 		options.addOption( "f", true, "Sets the output folder to export to" );
149 		options.addOption( "t", true, "Sets the soapui-settings.xml file to use" );
150 		return options;
151 	}
152 
153 	public SoapUILoadTestRunner()
154 	{
155 		this( TITLE );
156 	}
157 
158 	public SoapUILoadTestRunner( String title )
159 	{
160 		super( title );
161 	}
162 	
163 	public void setLoadTest(String loadTest)
164 	{
165 		this.loadTest = loadTest;
166 	}
167 
168 	public void setOutputFolder(String outputFolder)
169 	{
170 		this.outputFolder = outputFolder;
171 	}
172 
173 	public void setPrintReport(boolean printReport)
174 	{
175 		this.printReport = printReport;
176 	}
177 	
178 	/***
179 	 * Sets the host to use by all test-requests, the existing endpoint port and path will be used
180 	 * 
181 	 * @param host the host to use by all requests
182 	 */
183 
184 	public void setHost(String host)
185 	{
186 		this.host = host;
187 	}
188 
189 	/***
190 	 * Sets the domain to use for any authentications
191 	 * 
192 	 * @param domain the domain to use for any authentications
193 	 */
194 	
195 	public void setDomain(String domain)
196 	{
197 		this.domain = domain;
198 	}
199 
200 	/***
201 	 * Sets the password to use for any authentications
202 	 * 
203 	 * @param domain the password to use for any authentications
204 	 */
205 	
206 	public void setPassword(String password)
207 	{
208 		this.password = password;
209 	}
210 	
211 	/***
212 	 * Sets the WSS password-type to use for any authentications. Setting this will result
213 	 * in the addition of WS-Security UsernamePassword tokens to any outgoing request containing
214 	 * the specified username and password.
215 	 * 
216 	 * @param wssPasswordType the wss-password type to use, either 'Text' or 'Digest'
217 	 */
218 	
219 	public void setWssPasswordType( String wssPasswordType )
220 	{
221 		this.wssPasswordType = wssPasswordType;
222 	}
223 
224 	/***
225 	 * Sets the username to use for any authentications
226 	 * 
227 	 * @param domain the username to use for any authentications
228 	 */
229 	
230 	public void setUsername(String username)
231 	{
232 		this.username = username;
233 	}
234 	
235 	/***
236 	 * Runs the testcases as configured with setXXX methods
237 	 * 
238 	 * @throws Exception thrown if any tests fail
239 	 */
240 
241 	public boolean runRunner() throws Exception
242 	{
243 		if( SoapUI.getSettings().getBoolean( UISettings.DONT_DISABLE_GROOVY_LOG ))
244 		{
245 			initGroovyLog();
246 		}
247 		
248 		String projectFile = getProjectFile();
249 		
250 		WsdlProject project = new WsdlProject( projectFile );
251 		if( project.isDisabled() )
252 			throw new Exception( "Failed to load soapUI project file [" + projectFile + "]" );
253 		int suiteCount = 0;
254 		
255 		for( int c = 0; c < project.getTestSuiteCount(); c++ )
256 		{
257 			if( testSuite == null ||
258 				 project.getTestSuiteAt( c ).getName().equalsIgnoreCase( testSuite ))
259 			{
260 				runSuite( project.getTestSuiteAt( c ));
261 				suiteCount++;
262 			}
263 		}
264 		
265 		if( suiteCount == 0 )
266 		{
267 			log.warn( "No test-suites matched argument [" + testSuite + "]" );
268 		}
269 		else if( testCaseCount == 0 )
270 		{
271 			log.warn( "No test-cases matched argument [" + testCase + "]" );
272 		}
273 		else if( loadTestCount == 0 )
274 		{
275 			log.warn( "No load-tests matched argument [" + loadTest + "]" );
276 		}
277 		else if( !failedTests.isEmpty() )
278 		{
279 			log.info( failedTests.size() + " load tests failed:" );
280 			for( LoadTestRunner loadTestRunner : failedTests )
281 			{
282 				log.info( loadTestRunner.getLoadTest().getName() + ": " + loadTestRunner.getReason() );
283 			}
284 			
285 			throw new SoapUIException( "LoadTests failed" );
286 		}
287 		
288 		return true;
289 	}
290 
291 	/***
292 	 * Run tests in the specified TestSuite
293 	 *
294 	 * @param suite the TestSuite to run
295 	 */
296 	
297 	public void runSuite(TestSuite suite)
298 	{
299 		long start = System.currentTimeMillis();
300 		for( int c = 0; c < suite.getTestCaseCount(); c++ )
301 		{
302 			String name = suite.getTestCaseAt( c ).getName();
303 			if( testCase == null || 
304 				 name.equalsIgnoreCase( testCase ))
305 			{
306 				runTestCase( suite.getTestCaseAt( c ));
307 				testCaseCount++;
308 			}
309 			else
310 				log.info( "Skipping testcase [" + name + "], filter is [" + testCase + "]");
311 		}
312 		log.info( "soapUI suite [" + suite.getName() + "] finished in " + (System.currentTimeMillis()-start) + "ms" );
313 	}
314 
315 	/***
316 	 * Runs the specified TestCase
317 	 * 
318 	 * @param testCase the testcase to run
319 	 */
320 	
321 	private void runTestCase(TestCase testCase)
322 	{
323 		for( int c = 0; c < testCase.getLoadTestCount(); c++ )
324 		{
325 			String name = testCase.getLoadTestAt( c ).getName();
326 			if( loadTest == null || loadTest.equalsIgnoreCase( name ))
327 			{
328 				runWsdlLoadTest( (WsdlLoadTest) testCase.getLoadTestAt( c ));
329 				loadTestCount++;
330 			}
331 		}
332 	}
333 	
334 	/***
335 	 * Runs the specified LoadTest
336 	 * 
337 	 * @param loadTest the loadTest to run
338 	 */
339 
340 	private void runWsdlLoadTest(WsdlLoadTest loadTest)
341 	{
342 		try
343 		{
344 			log.info( "Running LoadTest [" + loadTest.getName() + "]" );
345 			if( limit >= 0 )
346 			{
347 				log.info( "Overriding limit [" + loadTest.getTestLimit() + "] with specified [" + limit + "]" );
348 				loadTest.setTestLimit( limit );
349 			}
350 			
351 			loadTest.addLoadTestRunListener(this);
352 			LoadTestRunner runner = loadTest.run();
353 			
354 			// wait for test to finish
355 			while (runner.getStatus() == LoadTestRunner.Status.RUNNING )
356 			{
357 				log.info( "LoadTest [" + loadTest.getName() + "] progress: " + runner.getProgress() + ", " + 
358 						runner.getRunningThreadCount() );
359 				Thread.sleep(1000);
360 			}
361 			
362 			log.info( "LoadTest [" + loadTest.getName() + "] finished with status " + runner.getStatus().toString() );
363 			
364 			if( printReport )
365 			{
366 				log.info( "Exporting log and statistics for LoadTest [" + loadTest.getName() + "]" );
367 				
368 				loadTest.getStatisticsModel().finish();
369 				
370 				exportLog( loadTest );
371 				exportStatistics( loadTest );
372 			}
373 		}
374 		catch (Exception e)
375 		{
376 			SoapUI.logError( e );
377 			log.error( e );
378 		}		
379 	}
380 
381 	private void exportStatistics(WsdlLoadTest loadTest) throws IOException
382 	{
383 		ExportStatisticsAction exportStatisticsAction = new ExportStatisticsAction( loadTest.getStatisticsModel() );
384 		String statisticsFileName = loadTest.getName() + "-statistics.txt";
385 		if( outputFolder != null )
386 		{
387 			ensureOutputFolder();
388 			statisticsFileName = outputFolder + File.separator + statisticsFileName;
389 		}
390 		
391 		int cnt = exportStatisticsAction.exportToFile( new File( statisticsFileName ));
392 		log.info( "Exported " + cnt + " statistics to [" + statisticsFileName + "]" );
393 	}
394 
395 	private void exportLog(WsdlLoadTest loadTest) throws IOException
396 	{
397 		// export log first
398 		LoadTestLog loadTestLog = loadTest.getLoadTestLog();
399 		ExportLoadTestLogAction exportLoadTestLogAction = new ExportLoadTestLogAction(loadTestLog, null);
400 		String logFileName = loadTest.getName() + "-log.txt";
401 		if( outputFolder != null )
402 		{
403 			ensureOutputFolder();
404 			logFileName = outputFolder + File.separator + logFileName;
405 		}
406 		
407 		int cnt = exportLoadTestLogAction.exportToFile( new File( logFileName ));
408 		log.info( "Exported " + cnt + " log items to [" + logFileName + "]" );
409 		
410 		int errorCnt = 0;
411 		for( int c = 0; c < loadTestLog.getSize(); c++ )
412 		{
413 			LoadTestLogEntry entry = (LoadTestLogEntry) loadTestLog.getElementAt( c );
414 			
415 			if( entry != null && entry.isError() )
416 			{
417 				String entryFileName = loadTest.getName() + "-error-" + errorCnt++ + "-entry.txt";
418 				if( outputFolder != null )
419 				{
420 					ensureOutputFolder();
421 					entryFileName = outputFolder + File.separator + entryFileName;
422 				}
423 
424 				try
425 				{
426 					entry.exportToFile( entryFileName );
427 				}
428 				catch (Exception e)
429 				{
430 					SoapUI.logError( e );
431 				}
432 			}
433 		}
434 		log.info( "Exported " + errorCnt + " error results" );
435 	}
436 
437 	private void ensureOutputFolder()
438 	{
439 		File folder = new File( outputFolder );
440 		if( !folder.exists())
441 			folder.mkdirs();
442 	}
443 
444 	/***
445 	 * Sets the testcase to run
446 	 * 
447 	 * @param testCase the testcase to run
448 	 */
449 	
450 	public void setTestCase(String testCase)
451 	{
452       this.testCase = testCase;
453 	}
454 	
455 	/***
456 	 * Sets the endpoint to use for all test requests
457 	 * 
458 	 * @param endpoint the endpoint to use for all test requests
459 	 */
460 	
461 	public void setEndpoint(String endpoint)
462 	{
463 		this.endpoint = endpoint.trim();
464 	}
465 	
466 	/***
467 	 * Sets the TestSuite to run. If not set all TestSuites in the specified project file are run
468 	 * 
469 	 * @param testSuite the testSuite to run.
470 	 */
471 
472 	public void setTestSuite(String testSuite)
473 	{
474 		this.testSuite = testSuite;
475 	}
476 	
477 	public void afterLoadTest(LoadTestRunner loadTestRunner, LoadTestRunContext context)
478 	{
479 		if( loadTestRunner.getStatus() == LoadTestRunner.Status.FAILED )
480 		{
481 			failedTests.add( loadTestRunner );
482 		}
483 	}
484 
485 	public void afterTestCase(LoadTestRunner loadTestRunner, LoadTestRunContext context, TestRunner testRunner, TestRunContext runContext)
486 	{
487 	}
488 
489 	public void afterTestStep(LoadTestRunner loadTestRunner, LoadTestRunContext context, TestRunner testRunner, TestRunContext runContext, TestStepResult testStepResult)
490 	{
491 	}
492 
493 	public void beforeLoadTest(LoadTestRunner loadTestRunner, LoadTestRunContext context)
494 	{
495 	}
496 
497 	public void beforeTestCase(LoadTestRunner loadTestRunner, LoadTestRunContext context, TestRunner testRunner, TestRunContext runContext)
498 	{
499 	}
500 
501 	public void beforeTestStep(LoadTestRunner loadTestRunner, LoadTestRunContext context, TestRunner testRunner, TestRunContext runContext, TestStep testStep)
502 	{
503 		if( testStep instanceof WsdlTestRequestStep )
504 		{
505 			WsdlTestRequestStep requestStep = (WsdlTestRequestStep) testStep;
506 			if( endpoint != null && endpoint.length() > 0 )
507 			{
508 				requestStep.getTestRequest().setEndpoint( endpoint );
509 			}
510 			
511 			if( host != null && host.length() > 0 )
512 			{
513 				try
514 				{
515 					String ep = Tools.replaceHost( requestStep.getTestRequest().getEndpoint(), host );
516 					requestStep.getTestRequest().setEndpoint( ep );
517 				}
518 				catch (Exception e)
519 				{
520 					log.error( "Failed to set host on endpoint", e );
521 				}				
522 			}
523 
524 			if( username != null && username.length() > 0 )
525 			{
526 				requestStep.getTestRequest().setUsername( username );
527 			}
528 			
529 			if( password != null && password.length() > 0 )
530 			{
531 				requestStep.getTestRequest().setPassword( password );
532 			}
533 			
534 			if( domain != null && domain.length() > 0 )
535 			{
536 				requestStep.getTestRequest().setDomain( domain );
537 			}
538 
539 			if( wssPasswordType != null && wssPasswordType.length() > 0 )
540 			{
541 				requestStep.getTestRequest().setWssPasswordType( wssPasswordType.equals( "Digest" ) ? 
542 							WsdlTestRequest.PW_TYPE_DIGEST : WsdlTestRequest.PW_TYPE_TEXT );
543 			}
544 		}
545 	}
546 
547 	public void loadTestStarted(LoadTestRunner loadTestRunner, LoadTestRunContext context)
548 	{
549 	}
550 
551 	public void loadTestStopped(LoadTestRunner loadTestRunner, LoadTestRunContext context)
552 	{
553 	}
554 }