View Javadoc

1   /*
2    *  soapUI, copyright (C) 2006 eviware.com 
3    *
4    *  soapUI is free software; you can redistribute it and/or modify it under the 
5    *  terms of the GNU Lesser General Public License as published by the Free Software Foundation; 
6    *  either version 2.1 of the License, or (at your option) any later version.
7    *
8    *  soapUI is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without 
9    *  even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 
10   *  See the GNU Lesser General Public License for more details at gnu.org.
11   */
12  
13  package com.eviware.soapui.tools;
14  
15  import java.io.File;
16  import java.io.IOException;
17  import java.io.OutputStreamWriter;
18  import java.util.ArrayList;
19  import java.util.List;
20  
21  import org.apache.commons.cli.CommandLine;
22  import org.apache.commons.cli.CommandLineParser;
23  import org.apache.commons.cli.HelpFormatter;
24  import org.apache.commons.cli.Options;
25  import org.apache.commons.cli.PosixParser;
26  import org.apache.log4j.ConsoleAppender;
27  import org.apache.log4j.Logger;
28  import org.apache.log4j.PatternLayout;
29  
30  import com.eviware.soapui.SoapUI;
31  import com.eviware.soapui.impl.wsdl.WsdlProject;
32  import com.eviware.soapui.impl.wsdl.loadtest.WsdlLoadTest;
33  import com.eviware.soapui.impl.wsdl.loadtest.data.actions.ExportLoadTestLogAction;
34  import com.eviware.soapui.impl.wsdl.loadtest.data.actions.ExportStatisticsAction;
35  import com.eviware.soapui.impl.wsdl.loadtest.log.LoadTestLog;
36  import com.eviware.soapui.impl.wsdl.loadtest.log.LoadTestLogEntry;
37  import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestRequest;
38  import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestRequestStep;
39  import com.eviware.soapui.model.testsuite.LoadTestRunContext;
40  import com.eviware.soapui.model.testsuite.LoadTestRunListener;
41  import com.eviware.soapui.model.testsuite.LoadTestRunner;
42  import com.eviware.soapui.model.testsuite.TestCase;
43  import com.eviware.soapui.model.testsuite.TestRunContext;
44  import com.eviware.soapui.model.testsuite.TestRunner;
45  import com.eviware.soapui.model.testsuite.TestStep;
46  import com.eviware.soapui.model.testsuite.TestStepResult;
47  import com.eviware.soapui.model.testsuite.TestSuite;
48  import com.eviware.soapui.monitor.TestMonitor;
49  import com.eviware.soapui.settings.UISettings;
50  import com.eviware.soapui.support.SoapUIException;
51  
52  /***
53   * Standalone test-runner used from maven-plugin, can also be used from command-line (see xdocs) or
54   * directly from other classes.
55   * <p>
56   * For standalone usage, set the project file (with setProjectFile) and other desired properties before
57   * calling run</p> 
58   * 
59   * @author Ole.Matzura
60   */
61  
62  public class SoapUILoadTestRunner implements LoadTestRunListener
63  {
64  	private String projectFile;
65  	private String testSuite;
66  	private String testCase;
67  	private String endpoint;
68  	private final static Logger log = Logger.getLogger( SoapUILoadTestRunner.class );
69  	private String domain;
70  	private String password;
71  	private String username;
72  	private String host;
73  	private String loadTest;
74  	private boolean printReport;
75  	private String outputFolder;
76  	private List<LoadTestRunner> failedTests = new ArrayList<LoadTestRunner>();
77  	private int testCaseCount;
78  	private int loadTestCount;
79  	private String wssPasswordType;
80  	
81  	public static String TITLE = "soapUI " + SoapUI.SOAPUI_VERSION + " LoadTest Runner";
82  	
83  	/***
84  	 * Runs the loadtests in the specified soapUI project file, see soapUI xdocs for details.
85  	 * 
86  	 * @param args
87  	 * @throws Exception
88  	 */
89  
90  	public static void main( String [] args) throws Exception
91  	{
92  		System.out.println( TITLE );
93  		SoapUILoadTestRunner runner = new SoapUILoadTestRunner();
94  		
95  		Options options = new Options();
96  		options.addOption( "e", true, "Sets the endpoint" );
97  		options.addOption( "s", true, "Sets the testsuite" );
98  		options.addOption( "c", true, "Sets the testcase" );
99  		options.addOption( "l", true, "Sets the loadtest" );
100 		options.addOption( "u", true, "Sets the username" );
101 		options.addOption( "p", true, "Sets the password" );
102 		options.addOption( "w", true, "Sets the WSS password type, either 'Text' or 'Digest'" );
103 		options.addOption( "d", true, "Sets the domain" );
104 		options.addOption( "h", true, "Sets the host" );
105 		options.addOption( "r", false, "Exports statistics and testlogs for each loadtest run" );
106 		options.addOption( "f", true, "Sets the output folder to export to" );
107 		options.addOption( "t", true, "Sets the soapui-settings.xml file to use" );
108 		
109 		CommandLineParser parser = new PosixParser();
110 		CommandLine cmd = parser.parse( options, args);
111 		
112 		String[] args2 = cmd.getArgs();
113 		if( args2.length != 1 )
114 		{
115 			HelpFormatter formatter = new HelpFormatter();
116 			formatter.printHelp( "loadtestrunner [options] <soapui-project-file>", options );
117 
118 			System.err.println( "Missing soapUI project file.." );
119 			return;
120 		}
121 		
122 		runner.setProjectFile( args2[0] );
123 		
124 		if( cmd.hasOption( "e"))
125 			runner.setEndpoint( cmd.getOptionValue( "e" ) );
126 		
127 		if( cmd.hasOption( "s"))
128 			runner.setTestSuite( cmd.getOptionValue( "s") );
129 
130 		if( cmd.hasOption( "c"))
131 			runner.setTestCase( cmd.getOptionValue( "c") );
132 
133 		if( cmd.hasOption( "l"))
134 			runner.setLoadTest( cmd.getOptionValue( "l") );
135 
136 		if( cmd.hasOption( "u"))
137 			runner.setUsername( cmd.getOptionValue( "u") );
138 
139 		if( cmd.hasOption( "p"))
140 			runner.setPassword( cmd.getOptionValue( "p") );
141 
142 		if( cmd.hasOption( "w"))
143 			runner.setWssPasswordType( cmd.getOptionValue( "w") );
144 		
145 		if( cmd.hasOption( "d"))
146 			runner.setDomain( cmd.getOptionValue( "d") );
147 
148 		if( cmd.hasOption( "h"))
149 			runner.setHost( cmd.getOptionValue( "h") );
150 		
151 		if( cmd.hasOption( "f"))
152 			runner.setOutputFolder( cmd.getOptionValue( "f") );
153 		
154 		if( cmd.hasOption( "t"))
155 			SoapUI.initSettings( cmd.getOptionValue( "t" ));
156 		
157 		runner.setPrintReport( cmd.hasOption( "r") );
158 		
159 		try
160 		{
161 			SoapUI.loadExtLibs();
162 			
163 			if( SoapUI.getSettings().getBoolean( UISettings.DONT_DISABLE_GROOVY_LOG ))
164 			{
165 				initGroovyLog();
166 			}
167 			
168 			runner.run();
169 			System.exit( 0 );
170 		}
171 		catch( SoapUIException e )
172 		{
173 			log.error( e.toString() );
174 			System.exit( 1 );
175 		}
176 		catch (Throwable e)
177 		{
178 			e.printStackTrace();
179 			System.exit( 2 );
180 		}		
181 	}
182 
183 	private static void initGroovyLog()
184 	{
185 		Logger logger = Logger.getLogger( "groovy.log" );
186 		
187 		ConsoleAppender appender = new ConsoleAppender();
188 		appender.setWriter( new OutputStreamWriter( System.out ));
189 		appender.setLayout( new PatternLayout( "%d{ABSOLUTE} %-5p [%c{1}] %m%n") );
190 		logger.addAppender( appender);
191 	}
192 	
193 	public SoapUILoadTestRunner()
194 	{
195 		SoapUI.setTestMonitor( new TestMonitor() );
196 	}
197 	
198 	public void setLoadTest(String loadTest)
199 	{
200 		this.loadTest = loadTest;
201 	}
202 
203 	public void setOutputFolder(String outputFolder)
204 	{
205 		this.outputFolder = outputFolder;
206 	}
207 
208 	public void setPrintReport(boolean printReport)
209 	{
210 		this.printReport = printReport;
211 	}
212 	
213 	/***
214 	 * Sets the host to use by all test-requests, the existing endpoint port and path will be used
215 	 * 
216 	 * @param host the host to use by all requests
217 	 */
218 
219 	public void setHost(String host)
220 	{
221 		this.host = host;
222 	}
223 
224 	/***
225 	 * Sets the domain to use for any authentications
226 	 * 
227 	 * @param domain the domain to use for any authentications
228 	 */
229 	
230 	public void setDomain(String domain)
231 	{
232 		log.info( "Setting domain to [" + domain + "]" );
233 		this.domain = domain;
234 	}
235 
236 	/***
237 	 * Sets the password to use for any authentications
238 	 * 
239 	 * @param domain the password to use for any authentications
240 	 */
241 	
242 	public void setPassword(String password)
243 	{
244 		log.info( "Setting password to [" + password + "]" );
245 		this.password = password;
246 	}
247 	
248 	/***
249 	 * Sets the WSS password-type to use for any authentications. Setting this will result
250 	 * in the addition of WS-Security UsernamePassword tokens to any outgoing request containing
251 	 * the specified username and password.
252 	 * 
253 	 * @param wssPasswordType the wss-password type to use, either 'Text' or 'Digest'
254 	 */
255 	
256 	public void setWssPasswordType( String wssPasswordType )
257 	{
258 		this.wssPasswordType = wssPasswordType;
259 	}
260 
261 	/***
262 	 * Sets the username to use for any authentications
263 	 * 
264 	 * @param domain the username to use for any authentications
265 	 */
266 	
267 	public void setUsername(String username)
268 	{
269 		log.info( "Setting username to [" + username + "]" );
270 		this.username = username;
271 	}
272 	
273 	/***
274 	 * Runs the testcases as configured with setXXX methods
275 	 * 
276 	 * @throws Exception thrown if any tests fail
277 	 */
278 
279 	public void run() throws Exception
280 	{
281 		if( !new File( projectFile ).exists() )
282 			throw new SoapUIException( "soapUI project file [" + projectFile + "] not found" );
283 		
284 		WsdlProject project = new WsdlProject( projectFile, null );
285 		int suiteCount = 0;
286 		
287 		for( int c = 0; c < project.getTestSuiteCount(); c++ )
288 		{
289 			if( testSuite == null ||
290 				 project.getTestSuiteAt( c ).getName().equalsIgnoreCase( testSuite ))
291 			{
292 				runSuite( project.getTestSuiteAt( c ));
293 				suiteCount++;
294 			}
295 		}
296 		
297 		if( suiteCount == 0 )
298 		{
299 			log.warn( "No test-suites matched argument [" + testSuite + "]" );
300 		}
301 		else if( testCaseCount == 0 )
302 		{
303 			log.warn( "No test-cases matched argument [" + testCase + "]" );
304 		}
305 		else if( loadTestCount == 0 )
306 		{
307 			log.warn( "No load-tests matched argument [" + loadTest + "]" );
308 		}
309 		else if( !failedTests.isEmpty() )
310 		{
311 			log.info( failedTests.size() + " load tests failed:" );
312 			for( LoadTestRunner loadTestRunner : failedTests )
313 			{
314 				log.info( loadTestRunner.getLoadTest().getName() + ": " + loadTestRunner.getReason() );
315 			}
316 			
317 			throw new SoapUIException( "LoadTests failed" );
318 		}
319 	}
320 
321 	/***
322 	 * Run tests in the specified TestSuite
323 	 *
324 	 * @param suite the TestSuite to run
325 	 */
326 	
327 	public void runSuite(TestSuite suite)
328 	{
329 		long start = System.currentTimeMillis();
330 		for( int c = 0; c < suite.getTestCaseCount(); c++ )
331 		{
332 			String name = suite.getTestCaseAt( c ).getName();
333 			if( testCase == null || 
334 				 name.equalsIgnoreCase( testCase ))
335 			{
336 				runTestCase( suite.getTestCaseAt( c ));
337 				testCaseCount++;
338 			}
339 			else
340 				log.info( "Skipping testcase [" + name + "], filter is [" + testCase + "]");
341 		}
342 		log.info( "soapUI suite [" + suite.getName() + "] finished in " + (System.currentTimeMillis()-start) + "ms" );
343 	}
344 
345 	/***
346 	 * Runs the specified TestCase
347 	 * 
348 	 * @param testCase the testcase to run
349 	 */
350 	
351 	private void runTestCase(TestCase testCase)
352 	{
353 		for( int c = 0; c < testCase.getLoadTestCount(); c++ )
354 		{
355 			String name = testCase.getLoadTestAt( c ).getName();
356 			if( loadTest == null || loadTest.equalsIgnoreCase( name ))
357 			{
358 				runWsdlLoadTest( (WsdlLoadTest) testCase.getLoadTestAt( c ));
359 				loadTestCount++;
360 			}
361 		}
362 	}
363 	
364 	/***
365 	 * Runs the specified LoadTest
366 	 * 
367 	 * @param loadTest the loadTest to run
368 	 */
369 
370 	private void runWsdlLoadTest(WsdlLoadTest loadTest)
371 	{
372 		try
373 		{
374 			log.info( "Running LoadTest [" + loadTest.getName() + "]" );
375 			loadTest.addLoadTestRunListener(this);
376 			LoadTestRunner runner = loadTest.run();
377 			
378 			// wait for test to finish
379 			while (runner.getStatus() == LoadTestRunner.Status.RUNNING )
380 			{
381 				log.info( "LoadTest [" + loadTest.getName() + "] progress: " + runner.getProgress() + ", " + 
382 						runner.getRunningThreadCount() );
383 				Thread.sleep(1000);
384 			}
385 			
386 			log.info( "LoadTest [" + loadTest.getName() + "] finished with status " + runner.getStatus().toString() );
387 			
388 			if( printReport )
389 			{
390 				log.info( "Exporting log and statistics for LoadTest [" + loadTest.getName() + "]" );
391 				
392 				loadTest.getStatisticsModel().finish();
393 				
394 				exportLog( loadTest );
395 				exportStatistics( loadTest );
396 			}
397 		}
398 		catch (Exception e)
399 		{
400 			e.printStackTrace();
401 			log.error( e );
402 		}		
403 	}
404 
405 	private void exportStatistics(WsdlLoadTest loadTest) throws IOException
406 	{
407 		ExportStatisticsAction exportStatisticsAction = new ExportStatisticsAction( loadTest.getStatisticsModel() );
408 		String statisticsFileName = loadTest.getName() + "-statistics.txt";
409 		if( outputFolder != null )
410 		{
411 			ensureOutputFolder();
412 			statisticsFileName = outputFolder + File.separator + statisticsFileName;
413 		}
414 		
415 		int cnt = exportStatisticsAction.exportToFile( new File( statisticsFileName ));
416 		log.info( "Exported " + cnt + " statistics to [" + statisticsFileName + "]" );
417 	}
418 
419 	private void exportLog(WsdlLoadTest loadTest) throws IOException
420 	{
421 		// export log first
422 		LoadTestLog loadTestLog = loadTest.getLoadTestLog();
423 		ExportLoadTestLogAction exportLoadTestLogAction = new ExportLoadTestLogAction(loadTestLog);
424 		String logFileName = loadTest.getName() + "-log.txt";
425 		if( outputFolder != null )
426 		{
427 			ensureOutputFolder();
428 			logFileName = outputFolder + File.separator + logFileName;
429 		}
430 		
431 		int cnt = exportLoadTestLogAction.exportToFile( new File( logFileName ));
432 		log.info( "Exported " + cnt + " log items to [" + logFileName + "]" );
433 		
434 		int errorCnt = 0;
435 		for( int c = 0; c < loadTestLog.getSize(); c++ )
436 		{
437 			LoadTestLogEntry entry = (LoadTestLogEntry) loadTestLog.getElementAt( c );
438 			
439 			if( entry != null && entry.isError() )
440 			{
441 				String entryFileName = loadTest.getName() + "-error-" + errorCnt++ + "-entry.txt";
442 				if( outputFolder != null )
443 				{
444 					ensureOutputFolder();
445 					entryFileName = outputFolder + File.separator + entryFileName;
446 				}
447 
448 				try
449 				{
450 					entry.exportToFile( entryFileName );
451 				}
452 				catch (Exception e)
453 				{
454 					e.printStackTrace();
455 				}
456 			}
457 		}
458 		log.info( "Exported " + errorCnt + " error results" );
459 	}
460 
461 	private void ensureOutputFolder()
462 	{
463 		File folder = new File( outputFolder );
464 		if( !folder.exists())
465 			folder.mkdirs();
466 	}
467 
468 	/***
469 	 * Sets the soapUI project file containing the tests to run
470 	 * 
471 	 * @param projectFile the soapUI project file containing the tests to run
472 	 */
473 	
474 	public void setProjectFile(String projectFile)
475 	{
476 		log.info( "setting projectFile to [" + projectFile + "]" );
477 		this.projectFile = projectFile;
478 	}
479 
480 	/***
481 	 * Sets the testcase to run
482 	 * 
483 	 * @param testCase the testcase to run
484 	 */
485 	
486 	public void setTestCase(String testCase)
487 	{
488 		log.info( "setting testCase to [" + testCase + "]" );
489       this.testCase = testCase;
490 	}
491 	
492 	/***
493 	 * Sets the endpoint to use for all test requests
494 	 * 
495 	 * @param endpoint the endpoint to use for all test requests
496 	 */
497 	
498 	public void setEndpoint(String endpoint)
499 	{
500 		log.info( "setting test endpoint to [" + endpoint+ "]" );
501 		this.endpoint = endpoint.trim();
502 	}
503 	
504 	/***
505 	 * Sets the TestSuite to run. If not set all TestSuites in the specified project file are run
506 	 * 
507 	 * @param testSuite the testSuite to run.
508 	 */
509 
510 	public void setTestSuite(String testSuite)
511 	{
512 	   log.info( "setting testSuite to [" + testSuite + "]" );
513 		this.testSuite = testSuite;
514 	}
515 	
516 	public void afterLoadTest(LoadTestRunner loadTestRunner, LoadTestRunContext context)
517 	{
518 		if( loadTestRunner.getStatus() == LoadTestRunner.Status.FAILED )
519 		{
520 			failedTests.add( loadTestRunner );
521 		}
522 	}
523 
524 	public void afterTestCase(LoadTestRunner loadTestRunner, LoadTestRunContext context, TestRunner testRunner, TestRunContext runContext)
525 	{
526 	}
527 
528 	public void afterTestStep(LoadTestRunner loadTestRunner, LoadTestRunContext context, TestRunner testRunner, TestRunContext runContext, TestStepResult testStepResult)
529 	{
530 	}
531 
532 	public void beforeLoadTest(LoadTestRunner loadTestRunner, LoadTestRunContext context)
533 	{
534 	}
535 
536 	public void beforeTestCase(LoadTestRunner loadTestRunner, LoadTestRunContext context, TestRunner testRunner, TestRunContext runContext)
537 	{
538 	}
539 
540 	public void beforeTestStep(LoadTestRunner loadTestRunner, LoadTestRunContext context, TestRunner testRunner, TestRunContext runContext, TestStep testStep)
541 	{
542 		if( testStep instanceof WsdlTestRequestStep )
543 		{
544 			WsdlTestRequestStep requestStep = (WsdlTestRequestStep) testStep;
545 			if( endpoint != null && endpoint.length() > 0 )
546 			{
547 				requestStep.getTestRequest().setEndpoint( endpoint );
548 			}
549 			
550 			if( host != null && host.length() > 0 )
551 			{
552 				try
553 				{
554 					String ep = SoapUITestCaseRunner.replaceHost( requestStep.getTestRequest().getEndpoint(), host );
555 					requestStep.getTestRequest().setEndpoint( ep );
556 				}
557 				catch (Exception e)
558 				{
559 					log.error( "Failed to set host on endpoint", e );
560 				}				
561 			}
562 
563 			if( username != null && username.length() > 0 )
564 			{
565 				requestStep.getTestRequest().setUsername( username );
566 			}
567 			
568 			if( password != null && password.length() > 0 )
569 			{
570 				requestStep.getTestRequest().setPassword( password );
571 			}
572 			
573 			if( domain != null && domain.length() > 0 )
574 			{
575 				requestStep.getTestRequest().setDomain( domain );
576 			}
577 
578 			if( wssPasswordType != null && wssPasswordType.length() > 0 )
579 			{
580 				requestStep.getTestRequest().setWssPasswordType( wssPasswordType.equals( "Digest" ) ? 
581 							WsdlTestRequest.PW_TYPE_DIGEST : WsdlTestRequest.PW_TYPE_TEXT );
582 			}
583 		}
584 	}
585 
586 	public void loadTestStarted(LoadTestRunner loadTestRunner, LoadTestRunContext context)
587 	{
588 	}
589 
590 	public void loadTestStopped(LoadTestRunner loadTestRunner, LoadTestRunContext context)
591 	{
592 	}
593 }