View Javadoc

1   /*
2    *  soapUI, copyright (C) 2004-2007 eviware.com 
3    *
4    *  soapUI is free software; you can redistribute it and/or modify it under the 
5    *  terms of version 2.1 of the GNU Lesser General Public License as published by 
6    *  the Free Software Foundation.
7    *
8    *  soapUI is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without 
9    *  even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 
10   *  See the GNU Lesser General Public License for more details at gnu.org.
11   */
12  
13  package com.eviware.soapui.tools;
14  
15  import java.io.File;
16  import java.io.IOException;
17  import java.util.ArrayList;
18  import java.util.List;
19  
20  import org.apache.commons.cli.CommandLine;
21  
22  import com.eviware.soapui.SoapUI;
23  import com.eviware.soapui.impl.wsdl.WsdlProject;
24  import com.eviware.soapui.impl.wsdl.loadtest.WsdlLoadTest;
25  import com.eviware.soapui.impl.wsdl.loadtest.data.actions.ExportLoadTestLogAction;
26  import com.eviware.soapui.impl.wsdl.loadtest.data.actions.ExportStatisticsAction;
27  import com.eviware.soapui.impl.wsdl.loadtest.log.LoadTestLog;
28  import com.eviware.soapui.impl.wsdl.loadtest.log.LoadTestLogEntry;
29  import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestRequest;
30  import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestRequestStep;
31  import com.eviware.soapui.model.testsuite.LoadTestRunContext;
32  import com.eviware.soapui.model.testsuite.LoadTestRunListener;
33  import com.eviware.soapui.model.testsuite.LoadTestRunner;
34  import com.eviware.soapui.model.testsuite.TestCase;
35  import com.eviware.soapui.model.testsuite.TestRunContext;
36  import com.eviware.soapui.model.testsuite.TestRunner;
37  import com.eviware.soapui.model.testsuite.TestStep;
38  import com.eviware.soapui.model.testsuite.TestStepResult;
39  import com.eviware.soapui.model.testsuite.TestSuite;
40  import com.eviware.soapui.settings.UISettings;
41  import com.eviware.soapui.support.SoapUIException;
42  import com.eviware.soapui.support.Tools;
43  
44  /***
45   * Standalone test-runner used from maven-plugin, can also be used from command-line (see xdocs) or
46   * directly from other classes.
47   * <p>
48   * For standalone usage, set the project file (with setProjectFile) and other desired properties before
49   * calling run</p> 
50   * 
51   * @author Ole.Matzura
52   */
53  
54  public class SoapUILoadTestRunner extends AbstractSoapUIRunner implements LoadTestRunListener
55  {
56  	private String testSuite;
57  	private String testCase;
58  	private String endpoint;
59  	private String domain;
60  	private String password;
61  	private String username;
62  	private String host;
63  	private String loadTest;
64  	private boolean printReport;
65  	private String outputFolder;
66  	private List<LoadTestRunner> failedTests = new ArrayList<LoadTestRunner>();
67  	private int testCaseCount;
68  	private int loadTestCount;
69  	private int limit = -1;
70  	private String wssPasswordType;
71  	
72  	public static String TITLE = "soapUI " + SoapUI.SOAPUI_VERSION + " LoadTest Runner";
73  	
74  	/***
75  	 * Runs the loadtests in the specified soapUI project file, see soapUI xdocs for details.
76  	 * 
77  	 * @param args
78  	 * @throws Exception
79  	 */
80  
81  	public static void main( String [] args ) 
82  	{
83  		new SoapUILoadTestRunner().runFromCommandLine( args );
84  	}
85  
86  	protected boolean processCommandLine( CommandLine cmd )
87  	{
88  		if( cmd.hasOption( "e"))
89  			setEndpoint( cmd.getOptionValue( "e" ) );
90  		
91  		if( cmd.hasOption( "s"))
92  			setTestSuite( cmd.getOptionValue( "s") );
93  
94  		if( cmd.hasOption( "c"))
95  			setTestCase( cmd.getOptionValue( "c") );
96  
97  		if( cmd.hasOption( "l"))
98  			setLoadTest( cmd.getOptionValue( "l") );
99  
100 		if( cmd.hasOption( "u"))
101 			setUsername( cmd.getOptionValue( "u") );
102 
103 		if( cmd.hasOption( "p"))
104 			setPassword( cmd.getOptionValue( "p") );
105 
106 		if( cmd.hasOption( "w"))
107 			setWssPasswordType( cmd.getOptionValue( "w") );
108 		
109 		if( cmd.hasOption( "d"))
110 			setDomain( cmd.getOptionValue( "d") );
111 
112 		if( cmd.hasOption( "h"))
113 			setHost( cmd.getOptionValue( "h") );
114 		
115 		if( cmd.hasOption( "m"))
116 			setLimit( Integer.parseInt( cmd.getOptionValue( "m") ));
117 		
118 		if( cmd.hasOption( "f"))
119 			setOutputFolder( cmd.getOptionValue( "f") );
120 		
121 		if( cmd.hasOption( "t"))
122 			setSettingsFile( cmd.getOptionValue( "t" ));
123 		
124 		setPrintReport( cmd.hasOption( "r") );
125 		
126 		return true;
127 	}
128 
129 	public void setLimit( int limit )
130 	{
131 		this.limit = limit;
132 	}
133 
134 	protected SoapUIOptions initCommandLineOptions()
135 	{
136 		SoapUIOptions options = new SoapUIOptions( "loadtestrunner" );
137 		options.addOption( "e", true, "Sets the endpoint" );
138 		options.addOption( "s", true, "Sets the testsuite" );
139 		options.addOption( "c", true, "Sets the testcase" );
140 		options.addOption( "l", true, "Sets the loadtest" );
141 		options.addOption( "u", true, "Sets the username" );
142 		options.addOption( "p", true, "Sets the password" );
143 		options.addOption( "w", true, "Sets the WSS password type, either 'Text' or 'Digest'" );
144 		options.addOption( "d", true, "Sets the domain" );
145 		options.addOption( "h", true, "Sets the host" );
146 		options.addOption( "m", true, "Overrides the LoadTest limit" );
147 		options.addOption( "r", false, "Exports statistics and testlogs for each LoadTest run" );
148 		options.addOption( "f", true, "Sets the output folder to export to" );
149 		options.addOption( "t", true, "Sets the soapui-settings.xml file to use" );
150 		return options;
151 	}
152 
153 	public SoapUILoadTestRunner()
154 	{
155 		this( TITLE );
156 	}
157 
158 	public SoapUILoadTestRunner( String title )
159 	{
160 		super( title );
161 	}
162 	
163 	public void setLoadTest(String loadTest)
164 	{
165 		this.loadTest = loadTest;
166 	}
167 
168 	public void setOutputFolder(String outputFolder)
169 	{
170 		this.outputFolder = outputFolder;
171 	}
172 
173 	public void setPrintReport(boolean printReport)
174 	{
175 		this.printReport = printReport;
176 	}
177 	
178 	/***
179 	 * Sets the host to use by all test-requests, the existing endpoint port and path will be used
180 	 * 
181 	 * @param host the host to use by all requests
182 	 */
183 
184 	public void setHost(String host)
185 	{
186 		this.host = host;
187 	}
188 
189 	/***
190 	 * Sets the domain to use for any authentications
191 	 * 
192 	 * @param domain the domain to use for any authentications
193 	 */
194 	
195 	public void setDomain(String domain)
196 	{
197 		log.info( "Setting domain to [" + domain + "]" );
198 		this.domain = domain;
199 	}
200 
201 	/***
202 	 * Sets the password to use for any authentications
203 	 * 
204 	 * @param domain the password to use for any authentications
205 	 */
206 	
207 	public void setPassword(String password)
208 	{
209 		log.info( "Setting password to [" + password + "]" );
210 		this.password = password;
211 	}
212 	
213 	/***
214 	 * Sets the WSS password-type to use for any authentications. Setting this will result
215 	 * in the addition of WS-Security UsernamePassword tokens to any outgoing request containing
216 	 * the specified username and password.
217 	 * 
218 	 * @param wssPasswordType the wss-password type to use, either 'Text' or 'Digest'
219 	 */
220 	
221 	public void setWssPasswordType( String wssPasswordType )
222 	{
223 		this.wssPasswordType = wssPasswordType;
224 	}
225 
226 	/***
227 	 * Sets the username to use for any authentications
228 	 * 
229 	 * @param domain the username to use for any authentications
230 	 */
231 	
232 	public void setUsername(String username)
233 	{
234 		log.info( "Setting username to [" + username + "]" );
235 		this.username = username;
236 	}
237 	
238 	/***
239 	 * Runs the testcases as configured with setXXX methods
240 	 * 
241 	 * @throws Exception thrown if any tests fail
242 	 */
243 
244 	public void runRunner() throws Exception
245 	{
246 		if( SoapUI.getSettings().getBoolean( UISettings.DONT_DISABLE_GROOVY_LOG ))
247 		{
248 			initGroovyLog();
249 		}
250 		
251 		String projectFile = getProjectFile();
252 		
253 		WsdlProject project = new WsdlProject( projectFile );
254 		if( project.isDisabled() )
255 			throw new Exception( "Failed to load soapUI project file [" + projectFile + "]" );
256 		int suiteCount = 0;
257 		
258 		for( int c = 0; c < project.getTestSuiteCount(); c++ )
259 		{
260 			if( testSuite == null ||
261 				 project.getTestSuiteAt( c ).getName().equalsIgnoreCase( testSuite ))
262 			{
263 				runSuite( project.getTestSuiteAt( c ));
264 				suiteCount++;
265 			}
266 		}
267 		
268 		if( suiteCount == 0 )
269 		{
270 			log.warn( "No test-suites matched argument [" + testSuite + "]" );
271 		}
272 		else if( testCaseCount == 0 )
273 		{
274 			log.warn( "No test-cases matched argument [" + testCase + "]" );
275 		}
276 		else if( loadTestCount == 0 )
277 		{
278 			log.warn( "No load-tests matched argument [" + loadTest + "]" );
279 		}
280 		else if( !failedTests.isEmpty() )
281 		{
282 			log.info( failedTests.size() + " load tests failed:" );
283 			for( LoadTestRunner loadTestRunner : failedTests )
284 			{
285 				log.info( loadTestRunner.getLoadTest().getName() + ": " + loadTestRunner.getReason() );
286 			}
287 			
288 			throw new SoapUIException( "LoadTests failed" );
289 		}
290 	}
291 
292 	/***
293 	 * Run tests in the specified TestSuite
294 	 *
295 	 * @param suite the TestSuite to run
296 	 */
297 	
298 	public void runSuite(TestSuite suite)
299 	{
300 		long start = System.currentTimeMillis();
301 		for( int c = 0; c < suite.getTestCaseCount(); c++ )
302 		{
303 			String name = suite.getTestCaseAt( c ).getName();
304 			if( testCase == null || 
305 				 name.equalsIgnoreCase( testCase ))
306 			{
307 				runTestCase( suite.getTestCaseAt( c ));
308 				testCaseCount++;
309 			}
310 			else
311 				log.info( "Skipping testcase [" + name + "], filter is [" + testCase + "]");
312 		}
313 		log.info( "soapUI suite [" + suite.getName() + "] finished in " + (System.currentTimeMillis()-start) + "ms" );
314 	}
315 
316 	/***
317 	 * Runs the specified TestCase
318 	 * 
319 	 * @param testCase the testcase to run
320 	 */
321 	
322 	private void runTestCase(TestCase testCase)
323 	{
324 		for( int c = 0; c < testCase.getLoadTestCount(); c++ )
325 		{
326 			String name = testCase.getLoadTestAt( c ).getName();
327 			if( loadTest == null || loadTest.equalsIgnoreCase( name ))
328 			{
329 				runWsdlLoadTest( (WsdlLoadTest) testCase.getLoadTestAt( c ));
330 				loadTestCount++;
331 			}
332 		}
333 	}
334 	
335 	/***
336 	 * Runs the specified LoadTest
337 	 * 
338 	 * @param loadTest the loadTest to run
339 	 */
340 
341 	private void runWsdlLoadTest(WsdlLoadTest loadTest)
342 	{
343 		try
344 		{
345 			log.info( "Running LoadTest [" + loadTest.getName() + "]" );
346 			if( limit >= 0 )
347 			{
348 				log.info( "Overriding limit [" + loadTest.getTestLimit() + "] with specified [" + limit + "]" );
349 				loadTest.setTestLimit( limit );
350 			}
351 			
352 			loadTest.addLoadTestRunListener(this);
353 			LoadTestRunner runner = loadTest.run();
354 			
355 			// wait for test to finish
356 			while (runner.getStatus() == LoadTestRunner.Status.RUNNING )
357 			{
358 				log.info( "LoadTest [" + loadTest.getName() + "] progress: " + runner.getProgress() + ", " + 
359 						runner.getRunningThreadCount() );
360 				Thread.sleep(1000);
361 			}
362 			
363 			log.info( "LoadTest [" + loadTest.getName() + "] finished with status " + runner.getStatus().toString() );
364 			
365 			if( printReport )
366 			{
367 				log.info( "Exporting log and statistics for LoadTest [" + loadTest.getName() + "]" );
368 				
369 				loadTest.getStatisticsModel().finish();
370 				
371 				exportLog( loadTest );
372 				exportStatistics( loadTest );
373 			}
374 		}
375 		catch (Exception e)
376 		{
377 			SoapUI.logError( e );
378 			log.error( e );
379 		}		
380 	}
381 
382 	private void exportStatistics(WsdlLoadTest loadTest) throws IOException
383 	{
384 		ExportStatisticsAction exportStatisticsAction = new ExportStatisticsAction( loadTest.getStatisticsModel() );
385 		String statisticsFileName = loadTest.getName() + "-statistics.txt";
386 		if( outputFolder != null )
387 		{
388 			ensureOutputFolder();
389 			statisticsFileName = outputFolder + File.separator + statisticsFileName;
390 		}
391 		
392 		int cnt = exportStatisticsAction.exportToFile( new File( statisticsFileName ));
393 		log.info( "Exported " + cnt + " statistics to [" + statisticsFileName + "]" );
394 	}
395 
396 	private void exportLog(WsdlLoadTest loadTest) throws IOException
397 	{
398 		// export log first
399 		LoadTestLog loadTestLog = loadTest.getLoadTestLog();
400 		ExportLoadTestLogAction exportLoadTestLogAction = new ExportLoadTestLogAction(loadTestLog, null);
401 		String logFileName = loadTest.getName() + "-log.txt";
402 		if( outputFolder != null )
403 		{
404 			ensureOutputFolder();
405 			logFileName = outputFolder + File.separator + logFileName;
406 		}
407 		
408 		int cnt = exportLoadTestLogAction.exportToFile( new File( logFileName ));
409 		log.info( "Exported " + cnt + " log items to [" + logFileName + "]" );
410 		
411 		int errorCnt = 0;
412 		for( int c = 0; c < loadTestLog.getSize(); c++ )
413 		{
414 			LoadTestLogEntry entry = (LoadTestLogEntry) loadTestLog.getElementAt( c );
415 			
416 			if( entry != null && entry.isError() )
417 			{
418 				String entryFileName = loadTest.getName() + "-error-" + errorCnt++ + "-entry.txt";
419 				if( outputFolder != null )
420 				{
421 					ensureOutputFolder();
422 					entryFileName = outputFolder + File.separator + entryFileName;
423 				}
424 
425 				try
426 				{
427 					entry.exportToFile( entryFileName );
428 				}
429 				catch (Exception e)
430 				{
431 					SoapUI.logError( e );
432 				}
433 			}
434 		}
435 		log.info( "Exported " + errorCnt + " error results" );
436 	}
437 
438 	private void ensureOutputFolder()
439 	{
440 		File folder = new File( outputFolder );
441 		if( !folder.exists())
442 			folder.mkdirs();
443 	}
444 
445 	/***
446 	 * Sets the testcase to run
447 	 * 
448 	 * @param testCase the testcase to run
449 	 */
450 	
451 	public void setTestCase(String testCase)
452 	{
453       this.testCase = testCase;
454 	}
455 	
456 	/***
457 	 * Sets the endpoint to use for all test requests
458 	 * 
459 	 * @param endpoint the endpoint to use for all test requests
460 	 */
461 	
462 	public void setEndpoint(String endpoint)
463 	{
464 		this.endpoint = endpoint.trim();
465 	}
466 	
467 	/***
468 	 * Sets the TestSuite to run. If not set all TestSuites in the specified project file are run
469 	 * 
470 	 * @param testSuite the testSuite to run.
471 	 */
472 
473 	public void setTestSuite(String testSuite)
474 	{
475 		this.testSuite = testSuite;
476 	}
477 	
478 	public void afterLoadTest(LoadTestRunner loadTestRunner, LoadTestRunContext context)
479 	{
480 		if( loadTestRunner.getStatus() == LoadTestRunner.Status.FAILED )
481 		{
482 			failedTests.add( loadTestRunner );
483 		}
484 	}
485 
486 	public void afterTestCase(LoadTestRunner loadTestRunner, LoadTestRunContext context, TestRunner testRunner, TestRunContext runContext)
487 	{
488 	}
489 
490 	public void afterTestStep(LoadTestRunner loadTestRunner, LoadTestRunContext context, TestRunner testRunner, TestRunContext runContext, TestStepResult testStepResult)
491 	{
492 	}
493 
494 	public void beforeLoadTest(LoadTestRunner loadTestRunner, LoadTestRunContext context)
495 	{
496 	}
497 
498 	public void beforeTestCase(LoadTestRunner loadTestRunner, LoadTestRunContext context, TestRunner testRunner, TestRunContext runContext)
499 	{
500 	}
501 
502 	public void beforeTestStep(LoadTestRunner loadTestRunner, LoadTestRunContext context, TestRunner testRunner, TestRunContext runContext, TestStep testStep)
503 	{
504 		if( testStep instanceof WsdlTestRequestStep )
505 		{
506 			WsdlTestRequestStep requestStep = (WsdlTestRequestStep) testStep;
507 			if( endpoint != null && endpoint.length() > 0 )
508 			{
509 				requestStep.getTestRequest().setEndpoint( endpoint );
510 			}
511 			
512 			if( host != null && host.length() > 0 )
513 			{
514 				try
515 				{
516 					String ep = Tools.replaceHost( requestStep.getTestRequest().getEndpoint(), host );
517 					requestStep.getTestRequest().setEndpoint( ep );
518 				}
519 				catch (Exception e)
520 				{
521 					log.error( "Failed to set host on endpoint", e );
522 				}				
523 			}
524 
525 			if( username != null && username.length() > 0 )
526 			{
527 				requestStep.getTestRequest().setUsername( username );
528 			}
529 			
530 			if( password != null && password.length() > 0 )
531 			{
532 				requestStep.getTestRequest().setPassword( password );
533 			}
534 			
535 			if( domain != null && domain.length() > 0 )
536 			{
537 				requestStep.getTestRequest().setDomain( domain );
538 			}
539 
540 			if( wssPasswordType != null && wssPasswordType.length() > 0 )
541 			{
542 				requestStep.getTestRequest().setWssPasswordType( wssPasswordType.equals( "Digest" ) ? 
543 							WsdlTestRequest.PW_TYPE_DIGEST : WsdlTestRequest.PW_TYPE_TEXT );
544 			}
545 		}
546 	}
547 
548 	public void loadTestStarted(LoadTestRunner loadTestRunner, LoadTestRunContext context)
549 	{
550 	}
551 
552 	public void loadTestStopped(LoadTestRunner loadTestRunner, LoadTestRunContext context)
553 	{
554 	}
555 }