View Javadoc

1   /*
2    *  soapUI, copyright (C) 2004-2007 eviware.com 
3    *
4    *  soapUI is free software; you can redistribute it and/or modify it under the 
5    *  terms of version 2.1 of the GNU Lesser General Public License as published by 
6    *  the Free Software Foundation.
7    *
8    *  soapUI is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without 
9    *  even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 
10   *  See the GNU Lesser General Public License for more details at gnu.org.
11   */
12  
13  package com.eviware.soapui.tools;
14  
15  import java.io.File;
16  import java.io.IOException;
17  import java.io.OutputStreamWriter;
18  import java.util.ArrayList;
19  import java.util.List;
20  
21  import org.apache.commons.cli.CommandLine;
22  import org.apache.commons.cli.CommandLineParser;
23  import org.apache.commons.cli.HelpFormatter;
24  import org.apache.commons.cli.Options;
25  import org.apache.commons.cli.PosixParser;
26  import org.apache.log4j.ConsoleAppender;
27  import org.apache.log4j.Logger;
28  import org.apache.log4j.PatternLayout;
29  
30  import com.eviware.soapui.SoapUI;
31  import com.eviware.soapui.impl.wsdl.WsdlProject;
32  import com.eviware.soapui.impl.wsdl.loadtest.WsdlLoadTest;
33  import com.eviware.soapui.impl.wsdl.loadtest.data.actions.ExportLoadTestLogAction;
34  import com.eviware.soapui.impl.wsdl.loadtest.data.actions.ExportStatisticsAction;
35  import com.eviware.soapui.impl.wsdl.loadtest.log.LoadTestLog;
36  import com.eviware.soapui.impl.wsdl.loadtest.log.LoadTestLogEntry;
37  import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestRequest;
38  import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestRequestStep;
39  import com.eviware.soapui.model.testsuite.LoadTestRunContext;
40  import com.eviware.soapui.model.testsuite.LoadTestRunListener;
41  import com.eviware.soapui.model.testsuite.LoadTestRunner;
42  import com.eviware.soapui.model.testsuite.TestCase;
43  import com.eviware.soapui.model.testsuite.TestRunContext;
44  import com.eviware.soapui.model.testsuite.TestRunner;
45  import com.eviware.soapui.model.testsuite.TestStep;
46  import com.eviware.soapui.model.testsuite.TestStepResult;
47  import com.eviware.soapui.model.testsuite.TestSuite;
48  import com.eviware.soapui.monitor.TestMonitor;
49  import com.eviware.soapui.settings.UISettings;
50  import com.eviware.soapui.support.SoapUIException;
51  
52  /***
53   * Standalone test-runner used from maven-plugin, can also be used from command-line (see xdocs) or
54   * directly from other classes.
55   * <p>
56   * For standalone usage, set the project file (with setProjectFile) and other desired properties before
57   * calling run</p> 
58   * 
59   * @author Ole.Matzura
60   */
61  
62  public class SoapUILoadTestRunner implements LoadTestRunListener
63  {
64  	private String projectFile;
65  	private String testSuite;
66  	private String testCase;
67  	private String endpoint;
68  	private final static Logger log = Logger.getLogger( SoapUILoadTestRunner.class );
69  	private String domain;
70  	private String password;
71  	private String username;
72  	private String host;
73  	private String loadTest;
74  	private boolean printReport;
75  	private String outputFolder;
76  	private List<LoadTestRunner> failedTests = new ArrayList<LoadTestRunner>();
77  	private int testCaseCount;
78  	private int loadTestCount;
79  	private String wssPasswordType;
80  	
81  	public static String TITLE = "soapUI " + SoapUI.SOAPUI_VERSION + " LoadTest Runner";
82  	
83  	/***
84  	 * Runs the loadtests in the specified soapUI project file, see soapUI xdocs for details.
85  	 * 
86  	 * @param args
87  	 * @throws Exception
88  	 */
89  
90  	public static void main( String [] args) throws Exception
91  	{
92  		System.out.println( TITLE );
93  		SoapUI.initSoapUILog();
94  		SoapUILoadTestRunner runner = new SoapUILoadTestRunner();
95  		
96  		Options options = new Options();
97  		options.addOption( "e", true, "Sets the endpoint" );
98  		options.addOption( "s", true, "Sets the testsuite" );
99  		options.addOption( "c", true, "Sets the testcase" );
100 		options.addOption( "l", true, "Sets the loadtest" );
101 		options.addOption( "u", true, "Sets the username" );
102 		options.addOption( "p", true, "Sets the password" );
103 		options.addOption( "w", true, "Sets the WSS password type, either 'Text' or 'Digest'" );
104 		options.addOption( "d", true, "Sets the domain" );
105 		options.addOption( "h", true, "Sets the host" );
106 		options.addOption( "r", false, "Exports statistics and testlogs for each loadtest run" );
107 		options.addOption( "f", true, "Sets the output folder to export to" );
108 		options.addOption( "t", true, "Sets the soapui-settings.xml file to use" );
109 		
110 		CommandLineParser parser = new PosixParser();
111 		CommandLine cmd = parser.parse( options, args);
112 		
113 		String[] args2 = cmd.getArgs();
114 		if( args2.length != 1 )
115 		{
116 			HelpFormatter formatter = new HelpFormatter();
117 			formatter.printHelp( "loadtestrunner [options] <soapui-project-file>", options );
118 
119 			System.err.println( "Missing soapUI project file.." );
120 			return;
121 		}
122 		
123 		runner.setProjectFile( args2[0] );
124 		
125 		if( cmd.hasOption( "e"))
126 			runner.setEndpoint( cmd.getOptionValue( "e" ) );
127 		
128 		if( cmd.hasOption( "s"))
129 			runner.setTestSuite( cmd.getOptionValue( "s") );
130 
131 		if( cmd.hasOption( "c"))
132 			runner.setTestCase( cmd.getOptionValue( "c") );
133 
134 		if( cmd.hasOption( "l"))
135 			runner.setLoadTest( cmd.getOptionValue( "l") );
136 
137 		if( cmd.hasOption( "u"))
138 			runner.setUsername( cmd.getOptionValue( "u") );
139 
140 		if( cmd.hasOption( "p"))
141 			runner.setPassword( cmd.getOptionValue( "p") );
142 
143 		if( cmd.hasOption( "w"))
144 			runner.setWssPasswordType( cmd.getOptionValue( "w") );
145 		
146 		if( cmd.hasOption( "d"))
147 			runner.setDomain( cmd.getOptionValue( "d") );
148 
149 		if( cmd.hasOption( "h"))
150 			runner.setHost( cmd.getOptionValue( "h") );
151 		
152 		if( cmd.hasOption( "f"))
153 			runner.setOutputFolder( cmd.getOptionValue( "f") );
154 		
155 		if( cmd.hasOption( "t"))
156 			SoapUI.initSettings( cmd.getOptionValue( "t" ));
157 		
158 		runner.setPrintReport( cmd.hasOption( "r") );
159 		
160 		try
161 		{
162 			SoapUI.loadExtLibs();
163 			
164 			if( SoapUI.getSettings().getBoolean( UISettings.DONT_DISABLE_GROOVY_LOG ))
165 			{
166 				initGroovyLog();
167 			}
168 			
169 			runner.run();
170 			System.exit( 0 );
171 		}
172 		catch( SoapUIException e )
173 		{
174 			log.error( e.toString() );
175 			System.exit( 1 );
176 		}
177 		catch (Throwable e)
178 		{
179 			e.printStackTrace();
180 			System.exit( 2 );
181 		}		
182 	}
183 
184 	private static void initGroovyLog()
185 	{
186 		Logger logger = Logger.getLogger( "groovy.log" );
187 		
188 		ConsoleAppender appender = new ConsoleAppender();
189 		appender.setWriter( new OutputStreamWriter( System.out ));
190 		appender.setLayout( new PatternLayout( "%d{ABSOLUTE} %-5p [%c{1}] %m%n") );
191 		logger.addAppender( appender);
192 	}
193 	
194 	public SoapUILoadTestRunner()
195 	{
196 		SoapUI.setTestMonitor( new TestMonitor() );
197 	}
198 	
199 	public void setLoadTest(String loadTest)
200 	{
201 		this.loadTest = loadTest;
202 	}
203 
204 	public void setOutputFolder(String outputFolder)
205 	{
206 		this.outputFolder = outputFolder;
207 	}
208 
209 	public void setPrintReport(boolean printReport)
210 	{
211 		this.printReport = printReport;
212 	}
213 	
214 	/***
215 	 * Sets the host to use by all test-requests, the existing endpoint port and path will be used
216 	 * 
217 	 * @param host the host to use by all requests
218 	 */
219 
220 	public void setHost(String host)
221 	{
222 		this.host = host;
223 	}
224 
225 	/***
226 	 * Sets the domain to use for any authentications
227 	 * 
228 	 * @param domain the domain to use for any authentications
229 	 */
230 	
231 	public void setDomain(String domain)
232 	{
233 		log.info( "Setting domain to [" + domain + "]" );
234 		this.domain = domain;
235 	}
236 
237 	/***
238 	 * Sets the password to use for any authentications
239 	 * 
240 	 * @param domain the password to use for any authentications
241 	 */
242 	
243 	public void setPassword(String password)
244 	{
245 		log.info( "Setting password to [" + password + "]" );
246 		this.password = password;
247 	}
248 	
249 	/***
250 	 * Sets the WSS password-type to use for any authentications. Setting this will result
251 	 * in the addition of WS-Security UsernamePassword tokens to any outgoing request containing
252 	 * the specified username and password.
253 	 * 
254 	 * @param wssPasswordType the wss-password type to use, either 'Text' or 'Digest'
255 	 */
256 	
257 	public void setWssPasswordType( String wssPasswordType )
258 	{
259 		this.wssPasswordType = wssPasswordType;
260 	}
261 
262 	/***
263 	 * Sets the username to use for any authentications
264 	 * 
265 	 * @param domain the username to use for any authentications
266 	 */
267 	
268 	public void setUsername(String username)
269 	{
270 		log.info( "Setting username to [" + username + "]" );
271 		this.username = username;
272 	}
273 	
274 	/***
275 	 * Runs the testcases as configured with setXXX methods
276 	 * 
277 	 * @throws Exception thrown if any tests fail
278 	 */
279 
280 	public void run() throws Exception
281 	{
282 		if( !new File( projectFile ).exists() )
283 			throw new SoapUIException( "soapUI project file [" + projectFile + "] not found" );
284 		
285 		WsdlProject project = new WsdlProject( projectFile, null );
286 		int suiteCount = 0;
287 		
288 		for( int c = 0; c < project.getTestSuiteCount(); c++ )
289 		{
290 			if( testSuite == null ||
291 				 project.getTestSuiteAt( c ).getName().equalsIgnoreCase( testSuite ))
292 			{
293 				runSuite( project.getTestSuiteAt( c ));
294 				suiteCount++;
295 			}
296 		}
297 		
298 		if( suiteCount == 0 )
299 		{
300 			log.warn( "No test-suites matched argument [" + testSuite + "]" );
301 		}
302 		else if( testCaseCount == 0 )
303 		{
304 			log.warn( "No test-cases matched argument [" + testCase + "]" );
305 		}
306 		else if( loadTestCount == 0 )
307 		{
308 			log.warn( "No load-tests matched argument [" + loadTest + "]" );
309 		}
310 		else if( !failedTests.isEmpty() )
311 		{
312 			log.info( failedTests.size() + " load tests failed:" );
313 			for( LoadTestRunner loadTestRunner : failedTests )
314 			{
315 				log.info( loadTestRunner.getLoadTest().getName() + ": " + loadTestRunner.getReason() );
316 			}
317 			
318 			throw new SoapUIException( "LoadTests failed" );
319 		}
320 	}
321 
322 	/***
323 	 * Run tests in the specified TestSuite
324 	 *
325 	 * @param suite the TestSuite to run
326 	 */
327 	
328 	public void runSuite(TestSuite suite)
329 	{
330 		long start = System.currentTimeMillis();
331 		for( int c = 0; c < suite.getTestCaseCount(); c++ )
332 		{
333 			String name = suite.getTestCaseAt( c ).getName();
334 			if( testCase == null || 
335 				 name.equalsIgnoreCase( testCase ))
336 			{
337 				runTestCase( suite.getTestCaseAt( c ));
338 				testCaseCount++;
339 			}
340 			else
341 				log.info( "Skipping testcase [" + name + "], filter is [" + testCase + "]");
342 		}
343 		log.info( "soapUI suite [" + suite.getName() + "] finished in " + (System.currentTimeMillis()-start) + "ms" );
344 	}
345 
346 	/***
347 	 * Runs the specified TestCase
348 	 * 
349 	 * @param testCase the testcase to run
350 	 */
351 	
352 	private void runTestCase(TestCase testCase)
353 	{
354 		for( int c = 0; c < testCase.getLoadTestCount(); c++ )
355 		{
356 			String name = testCase.getLoadTestAt( c ).getName();
357 			if( loadTest == null || loadTest.equalsIgnoreCase( name ))
358 			{
359 				runWsdlLoadTest( (WsdlLoadTest) testCase.getLoadTestAt( c ));
360 				loadTestCount++;
361 			}
362 		}
363 	}
364 	
365 	/***
366 	 * Runs the specified LoadTest
367 	 * 
368 	 * @param loadTest the loadTest to run
369 	 */
370 
371 	private void runWsdlLoadTest(WsdlLoadTest loadTest)
372 	{
373 		try
374 		{
375 			log.info( "Running LoadTest [" + loadTest.getName() + "]" );
376 			loadTest.addLoadTestRunListener(this);
377 			LoadTestRunner runner = loadTest.run();
378 			
379 			// wait for test to finish
380 			while (runner.getStatus() == LoadTestRunner.Status.RUNNING )
381 			{
382 				log.info( "LoadTest [" + loadTest.getName() + "] progress: " + runner.getProgress() + ", " + 
383 						runner.getRunningThreadCount() );
384 				Thread.sleep(1000);
385 			}
386 			
387 			log.info( "LoadTest [" + loadTest.getName() + "] finished with status " + runner.getStatus().toString() );
388 			
389 			if( printReport )
390 			{
391 				log.info( "Exporting log and statistics for LoadTest [" + loadTest.getName() + "]" );
392 				
393 				loadTest.getStatisticsModel().finish();
394 				
395 				exportLog( loadTest );
396 				exportStatistics( loadTest );
397 			}
398 		}
399 		catch (Exception e)
400 		{
401 			e.printStackTrace();
402 			log.error( e );
403 		}		
404 	}
405 
406 	private void exportStatistics(WsdlLoadTest loadTest) throws IOException
407 	{
408 		ExportStatisticsAction exportStatisticsAction = new ExportStatisticsAction( loadTest.getStatisticsModel() );
409 		String statisticsFileName = loadTest.getName() + "-statistics.txt";
410 		if( outputFolder != null )
411 		{
412 			ensureOutputFolder();
413 			statisticsFileName = outputFolder + File.separator + statisticsFileName;
414 		}
415 		
416 		int cnt = exportStatisticsAction.exportToFile( new File( statisticsFileName ));
417 		log.info( "Exported " + cnt + " statistics to [" + statisticsFileName + "]" );
418 	}
419 
420 	private void exportLog(WsdlLoadTest loadTest) throws IOException
421 	{
422 		// export log first
423 		LoadTestLog loadTestLog = loadTest.getLoadTestLog();
424 		ExportLoadTestLogAction exportLoadTestLogAction = new ExportLoadTestLogAction(loadTestLog);
425 		String logFileName = loadTest.getName() + "-log.txt";
426 		if( outputFolder != null )
427 		{
428 			ensureOutputFolder();
429 			logFileName = outputFolder + File.separator + logFileName;
430 		}
431 		
432 		int cnt = exportLoadTestLogAction.exportToFile( new File( logFileName ));
433 		log.info( "Exported " + cnt + " log items to [" + logFileName + "]" );
434 		
435 		int errorCnt = 0;
436 		for( int c = 0; c < loadTestLog.getSize(); c++ )
437 		{
438 			LoadTestLogEntry entry = (LoadTestLogEntry) loadTestLog.getElementAt( c );
439 			
440 			if( entry != null && entry.isError() )
441 			{
442 				String entryFileName = loadTest.getName() + "-error-" + errorCnt++ + "-entry.txt";
443 				if( outputFolder != null )
444 				{
445 					ensureOutputFolder();
446 					entryFileName = outputFolder + File.separator + entryFileName;
447 				}
448 
449 				try
450 				{
451 					entry.exportToFile( entryFileName );
452 				}
453 				catch (Exception e)
454 				{
455 					e.printStackTrace();
456 				}
457 			}
458 		}
459 		log.info( "Exported " + errorCnt + " error results" );
460 	}
461 
462 	private void ensureOutputFolder()
463 	{
464 		File folder = new File( outputFolder );
465 		if( !folder.exists())
466 			folder.mkdirs();
467 	}
468 
469 	/***
470 	 * Sets the soapUI project file containing the tests to run
471 	 * 
472 	 * @param projectFile the soapUI project file containing the tests to run
473 	 */
474 	
475 	public void setProjectFile(String projectFile)
476 	{
477 		log.info( "setting projectFile to [" + projectFile + "]" );
478 		this.projectFile = projectFile;
479 	}
480 
481 	/***
482 	 * Sets the testcase to run
483 	 * 
484 	 * @param testCase the testcase to run
485 	 */
486 	
487 	public void setTestCase(String testCase)
488 	{
489 		log.info( "setting testCase to [" + testCase + "]" );
490       this.testCase = testCase;
491 	}
492 	
493 	/***
494 	 * Sets the endpoint to use for all test requests
495 	 * 
496 	 * @param endpoint the endpoint to use for all test requests
497 	 */
498 	
499 	public void setEndpoint(String endpoint)
500 	{
501 		log.info( "setting test endpoint to [" + endpoint+ "]" );
502 		this.endpoint = endpoint.trim();
503 	}
504 	
505 	/***
506 	 * Sets the TestSuite to run. If not set all TestSuites in the specified project file are run
507 	 * 
508 	 * @param testSuite the testSuite to run.
509 	 */
510 
511 	public void setTestSuite(String testSuite)
512 	{
513 	   log.info( "setting testSuite to [" + testSuite + "]" );
514 		this.testSuite = testSuite;
515 	}
516 	
517 	public void afterLoadTest(LoadTestRunner loadTestRunner, LoadTestRunContext context)
518 	{
519 		if( loadTestRunner.getStatus() == LoadTestRunner.Status.FAILED )
520 		{
521 			failedTests.add( loadTestRunner );
522 		}
523 	}
524 
525 	public void afterTestCase(LoadTestRunner loadTestRunner, LoadTestRunContext context, TestRunner testRunner, TestRunContext runContext)
526 	{
527 	}
528 
529 	public void afterTestStep(LoadTestRunner loadTestRunner, LoadTestRunContext context, TestRunner testRunner, TestRunContext runContext, TestStepResult testStepResult)
530 	{
531 	}
532 
533 	public void beforeLoadTest(LoadTestRunner loadTestRunner, LoadTestRunContext context)
534 	{
535 	}
536 
537 	public void beforeTestCase(LoadTestRunner loadTestRunner, LoadTestRunContext context, TestRunner testRunner, TestRunContext runContext)
538 	{
539 	}
540 
541 	public void beforeTestStep(LoadTestRunner loadTestRunner, LoadTestRunContext context, TestRunner testRunner, TestRunContext runContext, TestStep testStep)
542 	{
543 		if( testStep instanceof WsdlTestRequestStep )
544 		{
545 			WsdlTestRequestStep requestStep = (WsdlTestRequestStep) testStep;
546 			if( endpoint != null && endpoint.length() > 0 )
547 			{
548 				requestStep.getTestRequest().setEndpoint( endpoint );
549 			}
550 			
551 			if( host != null && host.length() > 0 )
552 			{
553 				try
554 				{
555 					String ep = SoapUITestCaseRunner.replaceHost( requestStep.getTestRequest().getEndpoint(), host );
556 					requestStep.getTestRequest().setEndpoint( ep );
557 				}
558 				catch (Exception e)
559 				{
560 					log.error( "Failed to set host on endpoint", e );
561 				}				
562 			}
563 
564 			if( username != null && username.length() > 0 )
565 			{
566 				requestStep.getTestRequest().setUsername( username );
567 			}
568 			
569 			if( password != null && password.length() > 0 )
570 			{
571 				requestStep.getTestRequest().setPassword( password );
572 			}
573 			
574 			if( domain != null && domain.length() > 0 )
575 			{
576 				requestStep.getTestRequest().setDomain( domain );
577 			}
578 
579 			if( wssPasswordType != null && wssPasswordType.length() > 0 )
580 			{
581 				requestStep.getTestRequest().setWssPasswordType( wssPasswordType.equals( "Digest" ) ? 
582 							WsdlTestRequest.PW_TYPE_DIGEST : WsdlTestRequest.PW_TYPE_TEXT );
583 			}
584 		}
585 	}
586 
587 	public void loadTestStarted(LoadTestRunner loadTestRunner, LoadTestRunContext context)
588 	{
589 	}
590 
591 	public void loadTestStopped(LoadTestRunner loadTestRunner, LoadTestRunContext context)
592 	{
593 	}
594 }