View Javadoc

1   /*
2    *  soapUI, copyright (C) 2004-2007 eviware.com 
3    *
4    *  soapUI is free software; you can redistribute it and/or modify it under the 
5    *  terms of version 2.1 of the GNU Lesser General Public License as published by 
6    *  the Free Software Foundation.
7    *
8    *  soapUI is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without 
9    *  even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 
10   *  See the GNU Lesser General Public License for more details at gnu.org.
11   */
12  
13  package com.eviware.soapui.tools;
14  
15  import java.io.File;
16  import java.io.IOException;
17  import java.util.ArrayList;
18  import java.util.List;
19  
20  import org.apache.commons.cli.CommandLine;
21  
22  import com.eviware.soapui.SoapUI;
23  import com.eviware.soapui.impl.wsdl.WsdlProject;
24  import com.eviware.soapui.impl.wsdl.loadtest.WsdlLoadTest;
25  import com.eviware.soapui.impl.wsdl.loadtest.data.actions.ExportLoadTestLogAction;
26  import com.eviware.soapui.impl.wsdl.loadtest.data.actions.ExportStatisticsAction;
27  import com.eviware.soapui.impl.wsdl.loadtest.log.LoadTestLog;
28  import com.eviware.soapui.impl.wsdl.loadtest.log.LoadTestLogEntry;
29  import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestRequest;
30  import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestRequestStep;
31  import com.eviware.soapui.model.testsuite.LoadTestRunContext;
32  import com.eviware.soapui.model.testsuite.LoadTestRunListener;
33  import com.eviware.soapui.model.testsuite.LoadTestRunner;
34  import com.eviware.soapui.model.testsuite.TestCase;
35  import com.eviware.soapui.model.testsuite.TestRunContext;
36  import com.eviware.soapui.model.testsuite.TestRunner;
37  import com.eviware.soapui.model.testsuite.TestStep;
38  import com.eviware.soapui.model.testsuite.TestStepResult;
39  import com.eviware.soapui.model.testsuite.TestSuite;
40  import com.eviware.soapui.settings.UISettings;
41  import com.eviware.soapui.support.SoapUIException;
42  import com.eviware.soapui.support.Tools;
43  
44  /***
45   * Standalone test-runner used from maven-plugin, can also be used from command-line (see xdocs) or
46   * directly from other classes.
47   * <p>
48   * For standalone usage, set the project file (with setProjectFile) and other desired properties before
49   * calling run</p> 
50   * 
51   * @author Ole.Matzura
52   */
53  
54  public class SoapUILoadTestRunner extends AbstractSoapUIRunner implements LoadTestRunListener
55  {
56  	private String testSuite;
57  	private String testCase;
58  	private String endpoint;
59  	private String domain;
60  	private String password;
61  	private String username;
62  	private String host;
63  	private String loadTest;
64  	private boolean printReport;
65  	private String outputFolder;
66  	private List<LoadTestRunner> failedTests = new ArrayList<LoadTestRunner>();
67  	private int testCaseCount;
68  	private int loadTestCount;
69  	private int limit = -1;
70  	private String wssPasswordType;
71  	private long threadCount = -1;
72  	
73  	public static String TITLE = "soapUI " + SoapUI.SOAPUI_VERSION + " LoadTest Runner";
74  	
75  	/***
76  	 * Runs the loadtests in the specified soapUI project file, see soapUI xdocs for details.
77  	 * 
78  	 * @param args
79  	 * @throws Exception
80  	 */
81  
82  	public static void main( String [] args ) 
83  	{
84  		new SoapUILoadTestRunner().runFromCommandLine( args );
85  	}
86  
87  	protected boolean processCommandLine( CommandLine cmd )
88  	{
89  		if( cmd.hasOption( "e"))
90  			setEndpoint( cmd.getOptionValue( "e" ) );
91  		
92  		if( cmd.hasOption( "s"))
93  			setTestSuite( getCommandLineOptionSubstSpace( cmd, "s") );
94  
95  		if( cmd.hasOption( "c"))
96  			setTestCase( cmd.getOptionValue( "c") );
97  
98  		if( cmd.hasOption( "l"))
99  			setLoadTest( cmd.getOptionValue( "l") );
100 
101 		if( cmd.hasOption( "u"))
102 			setUsername( cmd.getOptionValue( "u") );
103 
104 		if( cmd.hasOption( "p"))
105 			setPassword( cmd.getOptionValue( "p") );
106 
107 		if( cmd.hasOption( "w"))
108 			setWssPasswordType( cmd.getOptionValue( "w") );
109 		
110 		if( cmd.hasOption( "d"))
111 			setDomain( cmd.getOptionValue( "d") );
112 
113 		if( cmd.hasOption( "h"))
114 			setHost( cmd.getOptionValue( "h") );
115 		
116 		if( cmd.hasOption( "m"))
117 			setLimit( Integer.parseInt( cmd.getOptionValue( "m") ));
118 		
119 		if( cmd.hasOption( "h"))
120 			setThreadCount( Integer.parseInt( cmd.getOptionValue( "h") ));
121 		
122 		if( cmd.hasOption( "f"))
123 			setOutputFolder( getCommandLineOptionSubstSpace( cmd, "f") );
124 		
125 		if( cmd.hasOption( "t"))
126 			setSettingsFile( getCommandLineOptionSubstSpace( cmd, "t" ) );
127 		
128 		setPrintReport( cmd.hasOption( "r") );
129 		
130 		return true;
131 	}
132 
133 	public void setLimit( int limit )
134 	{
135 		this.limit = limit;
136 	}
137 	
138 	public void setThreadCount( long threadCount )
139 	{
140 		this.threadCount = threadCount;
141 	}
142 
143 	protected SoapUIOptions initCommandLineOptions()
144 	{
145 		SoapUIOptions options = new SoapUIOptions( "loadtestrunner" );
146 		options.addOption( "e", true, "Sets the endpoint" );
147 		options.addOption( "s", true, "Sets the testsuite" );
148 		options.addOption( "c", true, "Sets the testcase" );
149 		options.addOption( "l", true, "Sets the loadtest" );
150 		options.addOption( "u", true, "Sets the username" );
151 		options.addOption( "p", true, "Sets the password" );
152 		options.addOption( "w", true, "Sets the WSS password type, either 'Text' or 'Digest'" );
153 		options.addOption( "d", true, "Sets the domain" );
154 		options.addOption( "h", true, "Sets the host" );
155 		options.addOption( "m", true, "Overrides the LoadTest Limit" );
156 		options.addOption( "h", true, "Overrides the LoadTest ThreadCount" );
157 		options.addOption( "r", false, "Exports statistics and testlogs for each LoadTest run" );
158 		options.addOption( "f", true, "Sets the output folder to export to" );
159 		options.addOption( "t", true, "Sets the soapui-settings.xml file to use" );
160 		return options;
161 	}
162 
163 	public SoapUILoadTestRunner()
164 	{
165 		this( TITLE );
166 	}
167 
168 	public SoapUILoadTestRunner( String title )
169 	{
170 		super( title );
171 	}
172 	
173 	public void setLoadTest(String loadTest)
174 	{
175 		this.loadTest = loadTest;
176 	}
177 
178 	public void setOutputFolder(String outputFolder)
179 	{
180 		this.outputFolder = outputFolder;
181 	}
182 
183 	public void setPrintReport(boolean printReport)
184 	{
185 		this.printReport = printReport;
186 	}
187 	
188 	/***
189 	 * Sets the host to use by all test-requests, the existing endpoint port and path will be used
190 	 * 
191 	 * @param host the host to use by all requests
192 	 */
193 
194 	public void setHost(String host)
195 	{
196 		this.host = host;
197 	}
198 
199 	/***
200 	 * Sets the domain to use for any authentications
201 	 * 
202 	 * @param domain the domain to use for any authentications
203 	 */
204 	
205 	public void setDomain(String domain)
206 	{
207 		this.domain = domain;
208 	}
209 
210 	/***
211 	 * Sets the password to use for any authentications
212 	 * 
213 	 * @param domain the password to use for any authentications
214 	 */
215 	
216 	public void setPassword(String password)
217 	{
218 		this.password = password;
219 	}
220 	
221 	/***
222 	 * Sets the WSS password-type to use for any authentications. Setting this will result
223 	 * in the addition of WS-Security UsernamePassword tokens to any outgoing request containing
224 	 * the specified username and password.
225 	 * 
226 	 * @param wssPasswordType the wss-password type to use, either 'Text' or 'Digest'
227 	 */
228 	
229 	public void setWssPasswordType( String wssPasswordType )
230 	{
231 		this.wssPasswordType = wssPasswordType;
232 	}
233 
234 	/***
235 	 * Sets the username to use for any authentications
236 	 * 
237 	 * @param domain the username to use for any authentications
238 	 */
239 	
240 	public void setUsername(String username)
241 	{
242 		this.username = username;
243 	}
244 	
245 	/***
246 	 * Runs the testcases as configured with setXXX methods
247 	 * 
248 	 * @throws Exception thrown if any tests fail
249 	 */
250 
251 	public boolean runRunner() throws Exception
252 	{
253 		if( SoapUI.getSettings().getBoolean( UISettings.DONT_DISABLE_GROOVY_LOG ))
254 		{
255 			initGroovyLog();
256 		}
257 		
258 		String projectFile = getProjectFile();
259 		
260 		WsdlProject project = new WsdlProject( projectFile );
261 		if( project.isDisabled() )
262 			throw new Exception( "Failed to load soapUI project file [" + projectFile + "]" );
263 		int suiteCount = 0;
264 		
265 		for( int c = 0; c < project.getTestSuiteCount(); c++ )
266 		{
267 			if( testSuite == null ||
268 				 project.getTestSuiteAt( c ).getName().equalsIgnoreCase( testSuite ))
269 			{
270 				runSuite( project.getTestSuiteAt( c ));
271 				suiteCount++;
272 			}
273 		}
274 		
275 		if( suiteCount == 0 )
276 		{
277 			log.warn( "No test-suites matched argument [" + testSuite + "]" );
278 		}
279 		else if( testCaseCount == 0 )
280 		{
281 			log.warn( "No test-cases matched argument [" + testCase + "]" );
282 		}
283 		else if( loadTestCount == 0 )
284 		{
285 			log.warn( "No load-tests matched argument [" + loadTest + "]" );
286 		}
287 		else if( !failedTests.isEmpty() )
288 		{
289 			log.info( failedTests.size() + " load tests failed:" );
290 			for( LoadTestRunner loadTestRunner : failedTests )
291 			{
292 				log.info( loadTestRunner.getLoadTest().getName() + ": " + loadTestRunner.getReason() );
293 			}
294 			
295 			throw new SoapUIException( "LoadTests failed" );
296 		}
297 		
298 		return true;
299 	}
300 
301 	/***
302 	 * Run tests in the specified TestSuite
303 	 *
304 	 * @param suite the TestSuite to run
305 	 */
306 	
307 	public void runSuite(TestSuite suite)
308 	{
309 		long start = System.currentTimeMillis();
310 		for( int c = 0; c < suite.getTestCaseCount(); c++ )
311 		{
312 			String name = suite.getTestCaseAt( c ).getName();
313 			if( testCase == null || 
314 				 name.equalsIgnoreCase( testCase ))
315 			{
316 				runTestCase( suite.getTestCaseAt( c ));
317 				testCaseCount++;
318 			}
319 			else
320 				log.info( "Skipping testcase [" + name + "], filter is [" + testCase + "]");
321 		}
322 		log.info( "soapUI suite [" + suite.getName() + "] finished in " + (System.currentTimeMillis()-start) + "ms" );
323 	}
324 
325 	/***
326 	 * Runs the specified TestCase
327 	 * 
328 	 * @param testCase the testcase to run
329 	 */
330 	
331 	private void runTestCase(TestCase testCase)
332 	{
333 		for( int c = 0; c < testCase.getLoadTestCount(); c++ )
334 		{
335 			String name = testCase.getLoadTestAt( c ).getName();
336 			if( loadTest == null || loadTest.equalsIgnoreCase( name ))
337 			{
338 				runWsdlLoadTest( (WsdlLoadTest) testCase.getLoadTestAt( c ));
339 				loadTestCount++;
340 			}
341 		}
342 	}
343 	
344 	/***
345 	 * Runs the specified LoadTest
346 	 * 
347 	 * @param loadTest the loadTest to run
348 	 */
349 
350 	private void runWsdlLoadTest(WsdlLoadTest loadTest)
351 	{
352 		try
353 		{
354 			log.info( "Running LoadTest [" + loadTest.getName() + "]" );
355 			if( limit >= 0 )
356 			{
357 				log.info( "Overriding limit [" + loadTest.getTestLimit() + "] with specified [" + limit + "]" );
358 				loadTest.setTestLimit( limit );
359 			}
360 			
361 			if( threadCount >= 0 )
362 			{
363 				log.info( "Overriding threadCount [" + loadTest.getThreadCount() + "] with specified [" + threadCount + "]" );
364 				loadTest.setThreadCount( threadCount  );
365 			}
366 			
367 			loadTest.addLoadTestRunListener(this);
368 			LoadTestRunner runner = loadTest.run();
369 			
370 			// wait for test to finish
371 			while (runner.getStatus() == LoadTestRunner.Status.RUNNING )
372 			{
373 				log.info( "LoadTest [" + loadTest.getName() + "] progress: " + runner.getProgress() + ", " + 
374 						runner.getRunningThreadCount() );
375 				Thread.sleep(1000);
376 			}
377 			
378 			log.info( "LoadTest [" + loadTest.getName() + "] finished with status " + runner.getStatus().toString() );
379 			
380 			if( printReport )
381 			{
382 				log.info( "Exporting log and statistics for LoadTest [" + loadTest.getName() + "]" );
383 				
384 				loadTest.getStatisticsModel().finish();
385 				
386 				exportLog( loadTest );
387 				exportStatistics( loadTest );
388 			}
389 		}
390 		catch (Exception e)
391 		{
392 			SoapUI.logError( e );
393 			log.error( e );
394 		}		
395 	}
396 
397 	private void exportStatistics(WsdlLoadTest loadTest) throws IOException
398 	{
399 		ExportStatisticsAction exportStatisticsAction = new ExportStatisticsAction( loadTest.getStatisticsModel() );
400 		String statisticsFileName = loadTest.getName() + "-statistics.txt";
401 		if( outputFolder != null )
402 		{
403 			ensureOutputFolder();
404 			statisticsFileName = outputFolder + File.separator + statisticsFileName;
405 		}
406 		
407 		int cnt = exportStatisticsAction.exportToFile( new File( statisticsFileName ));
408 		log.info( "Exported " + cnt + " statistics to [" + statisticsFileName + "]" );
409 	}
410 
411 	private void exportLog(WsdlLoadTest loadTest) throws IOException
412 	{
413 		// export log first
414 		LoadTestLog loadTestLog = loadTest.getLoadTestLog();
415 		ExportLoadTestLogAction exportLoadTestLogAction = new ExportLoadTestLogAction(loadTestLog, null);
416 		String logFileName = loadTest.getName() + "-log.txt";
417 		if( outputFolder != null )
418 		{
419 			ensureOutputFolder();
420 			logFileName = outputFolder + File.separator + logFileName;
421 		}
422 		
423 		int cnt = exportLoadTestLogAction.exportToFile( new File( logFileName ));
424 		log.info( "Exported " + cnt + " log items to [" + logFileName + "]" );
425 		
426 		int errorCnt = 0;
427 		for( int c = 0; c < loadTestLog.getSize(); c++ )
428 		{
429 			LoadTestLogEntry entry = (LoadTestLogEntry) loadTestLog.getElementAt( c );
430 			
431 			if( entry != null && entry.isError() )
432 			{
433 				String entryFileName = loadTest.getName() + "-error-" + errorCnt++ + "-entry.txt";
434 				if( outputFolder != null )
435 				{
436 					ensureOutputFolder();
437 					entryFileName = outputFolder + File.separator + entryFileName;
438 				}
439 
440 				try
441 				{
442 					entry.exportToFile( entryFileName );
443 				}
444 				catch (Exception e)
445 				{
446 					SoapUI.logError( e );
447 				}
448 			}
449 		}
450 		log.info( "Exported " + errorCnt + " error results" );
451 	}
452 
453 	private void ensureOutputFolder()
454 	{
455 		File folder = new File( outputFolder );
456 		if( !folder.exists())
457 			folder.mkdirs();
458 	}
459 
460 	/***
461 	 * Sets the testcase to run
462 	 * 
463 	 * @param testCase the testcase to run
464 	 */
465 	
466 	public void setTestCase(String testCase)
467 	{
468       this.testCase = testCase;
469 	}
470 	
471 	/***
472 	 * Sets the endpoint to use for all test requests
473 	 * 
474 	 * @param endpoint the endpoint to use for all test requests
475 	 */
476 	
477 	public void setEndpoint(String endpoint)
478 	{
479 		this.endpoint = endpoint.trim();
480 	}
481 	
482 	/***
483 	 * Sets the TestSuite to run. If not set all TestSuites in the specified project file are run
484 	 * 
485 	 * @param testSuite the testSuite to run.
486 	 */
487 
488 	public void setTestSuite(String testSuite)
489 	{
490 		this.testSuite = testSuite;
491 	}
492 	
493 	public void afterLoadTest(LoadTestRunner loadTestRunner, LoadTestRunContext context)
494 	{
495 		if( loadTestRunner.getStatus() == LoadTestRunner.Status.FAILED )
496 		{
497 			failedTests.add( loadTestRunner );
498 		}
499 	}
500 
501 	public void afterTestCase(LoadTestRunner loadTestRunner, LoadTestRunContext context, TestRunner testRunner, TestRunContext runContext)
502 	{
503 	}
504 
505 	public void afterTestStep(LoadTestRunner loadTestRunner, LoadTestRunContext context, TestRunner testRunner, TestRunContext runContext, TestStepResult testStepResult)
506 	{
507 	}
508 
509 	public void beforeLoadTest(LoadTestRunner loadTestRunner, LoadTestRunContext context)
510 	{
511 	}
512 
513 	public void beforeTestCase(LoadTestRunner loadTestRunner, LoadTestRunContext context, TestRunner testRunner, TestRunContext runContext)
514 	{
515 	}
516 
517 	public void beforeTestStep(LoadTestRunner loadTestRunner, LoadTestRunContext context, TestRunner testRunner, TestRunContext runContext, TestStep testStep)
518 	{
519 		if( testStep instanceof WsdlTestRequestStep )
520 		{
521 			WsdlTestRequestStep requestStep = (WsdlTestRequestStep) testStep;
522 			if( endpoint != null && endpoint.length() > 0 )
523 			{
524 				requestStep.getTestRequest().setEndpoint( endpoint );
525 			}
526 			
527 			if( host != null && host.length() > 0 )
528 			{
529 				try
530 				{
531 					String ep = Tools.replaceHost( requestStep.getTestRequest().getEndpoint(), host );
532 					requestStep.getTestRequest().setEndpoint( ep );
533 				}
534 				catch (Exception e)
535 				{
536 					log.error( "Failed to set host on endpoint", e );
537 				}				
538 			}
539 
540 			if( username != null && username.length() > 0 )
541 			{
542 				requestStep.getTestRequest().setUsername( username );
543 			}
544 			
545 			if( password != null && password.length() > 0 )
546 			{
547 				requestStep.getTestRequest().setPassword( password );
548 			}
549 			
550 			if( domain != null && domain.length() > 0 )
551 			{
552 				requestStep.getTestRequest().setDomain( domain );
553 			}
554 
555 			if( wssPasswordType != null && wssPasswordType.length() > 0 )
556 			{
557 				requestStep.getTestRequest().setWssPasswordType( wssPasswordType.equals( "Digest" ) ? 
558 							WsdlTestRequest.PW_TYPE_DIGEST : WsdlTestRequest.PW_TYPE_TEXT );
559 			}
560 		}
561 	}
562 
563 	public void loadTestStarted(LoadTestRunner loadTestRunner, LoadTestRunContext context)
564 	{
565 	}
566 
567 	public void loadTestStopped(LoadTestRunner loadTestRunner, LoadTestRunContext context)
568 	{
569 	}
570 }