1
2
3
4
5
6
7
8
9
10
11
12
13 package com.eviware.soapui.tools;
14
15 import java.io.File;
16 import java.io.IOException;
17 import java.io.OutputStreamWriter;
18 import java.util.ArrayList;
19 import java.util.List;
20
21 import org.apache.commons.cli.CommandLine;
22 import org.apache.commons.cli.CommandLineParser;
23 import org.apache.commons.cli.HelpFormatter;
24 import org.apache.commons.cli.Options;
25 import org.apache.commons.cli.PosixParser;
26 import org.apache.log4j.ConsoleAppender;
27 import org.apache.log4j.Logger;
28 import org.apache.log4j.PatternLayout;
29
30 import com.eviware.soapui.SoapUI;
31 import com.eviware.soapui.impl.wsdl.WsdlProject;
32 import com.eviware.soapui.impl.wsdl.loadtest.WsdlLoadTest;
33 import com.eviware.soapui.impl.wsdl.loadtest.data.actions.ExportLoadTestLogAction;
34 import com.eviware.soapui.impl.wsdl.loadtest.data.actions.ExportStatisticsAction;
35 import com.eviware.soapui.impl.wsdl.loadtest.log.LoadTestLog;
36 import com.eviware.soapui.impl.wsdl.loadtest.log.LoadTestLogEntry;
37 import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestRequest;
38 import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestRequestStep;
39 import com.eviware.soapui.model.testsuite.LoadTestRunContext;
40 import com.eviware.soapui.model.testsuite.LoadTestRunListener;
41 import com.eviware.soapui.model.testsuite.LoadTestRunner;
42 import com.eviware.soapui.model.testsuite.TestCase;
43 import com.eviware.soapui.model.testsuite.TestRunContext;
44 import com.eviware.soapui.model.testsuite.TestRunner;
45 import com.eviware.soapui.model.testsuite.TestStep;
46 import com.eviware.soapui.model.testsuite.TestStepResult;
47 import com.eviware.soapui.model.testsuite.TestSuite;
48 import com.eviware.soapui.monitor.TestMonitor;
49 import com.eviware.soapui.settings.UISettings;
50 import com.eviware.soapui.support.SoapUIException;
51
52 /***
53 * Standalone test-runner used from maven-plugin, can also be used from command-line (see xdocs) or
54 * directly from other classes.
55 * <p>
56 * For standalone usage, set the project file (with setProjectFile) and other desired properties before
57 * calling run</p>
58 *
59 * @author Ole.Matzura
60 */
61
62 public class SoapUILoadTestRunner implements LoadTestRunListener
63 {
64 private String projectFile;
65 private String testSuite;
66 private String testCase;
67 private String endpoint;
68 private final static Logger log = Logger.getLogger( SoapUILoadTestRunner.class );
69 private String domain;
70 private String password;
71 private String username;
72 private String host;
73 private String loadTest;
74 private boolean printReport;
75 private String outputFolder;
76 private List<LoadTestRunner> failedTests = new ArrayList<LoadTestRunner>();
77 private int testCaseCount;
78 private int loadTestCount;
79 private String wssPasswordType;
80
81 public static String TITLE = "soapUI " + SoapUI.SOAPUI_VERSION + " LoadTest Runner";
82
83 /***
84 * Runs the loadtests in the specified soapUI project file, see soapUI xdocs for details.
85 *
86 * @param args
87 * @throws Exception
88 */
89
90 public static void main( String [] args) throws Exception
91 {
92 System.out.println( TITLE );
93 SoapUILoadTestRunner runner = new SoapUILoadTestRunner();
94
95 Options options = new Options();
96 options.addOption( "e", true, "Sets the endpoint" );
97 options.addOption( "s", true, "Sets the testsuite" );
98 options.addOption( "c", true, "Sets the testcase" );
99 options.addOption( "l", true, "Sets the loadtest" );
100 options.addOption( "u", true, "Sets the username" );
101 options.addOption( "p", true, "Sets the password" );
102 options.addOption( "w", true, "Sets the WSS password type, either 'Text' or 'Digest'" );
103 options.addOption( "d", true, "Sets the domain" );
104 options.addOption( "h", true, "Sets the host" );
105 options.addOption( "r", false, "Exports statistics and testlogs for each loadtest run" );
106 options.addOption( "f", true, "Sets the output folder to export to" );
107 options.addOption( "t", true, "Sets the soapui-settings.xml file to use" );
108
109 CommandLineParser parser = new PosixParser();
110 CommandLine cmd = parser.parse( options, args);
111
112 String[] args2 = cmd.getArgs();
113 if( args2.length != 1 )
114 {
115 HelpFormatter formatter = new HelpFormatter();
116 formatter.printHelp( "loadtestrunner [options] <soapui-project-file>", options );
117
118 System.err.println( "Missing soapUI project file.." );
119 return;
120 }
121
122 runner.setProjectFile( args2[0] );
123
124 if( cmd.hasOption( "e"))
125 runner.setEndpoint( cmd.getOptionValue( "e" ) );
126
127 if( cmd.hasOption( "s"))
128 runner.setTestSuite( cmd.getOptionValue( "s") );
129
130 if( cmd.hasOption( "c"))
131 runner.setTestCase( cmd.getOptionValue( "c") );
132
133 if( cmd.hasOption( "l"))
134 runner.setLoadTest( cmd.getOptionValue( "l") );
135
136 if( cmd.hasOption( "u"))
137 runner.setUsername( cmd.getOptionValue( "u") );
138
139 if( cmd.hasOption( "p"))
140 runner.setPassword( cmd.getOptionValue( "p") );
141
142 if( cmd.hasOption( "w"))
143 runner.setWssPasswordType( cmd.getOptionValue( "w") );
144
145 if( cmd.hasOption( "d"))
146 runner.setDomain( cmd.getOptionValue( "d") );
147
148 if( cmd.hasOption( "h"))
149 runner.setHost( cmd.getOptionValue( "h") );
150
151 if( cmd.hasOption( "f"))
152 runner.setOutputFolder( cmd.getOptionValue( "f") );
153
154 if( cmd.hasOption( "t"))
155 SoapUI.initSettings( cmd.getOptionValue( "t" ));
156
157 runner.setPrintReport( cmd.hasOption( "r") );
158
159 try
160 {
161 SoapUI.loadExtLibs();
162
163 if( SoapUI.getSettings().getBoolean( UISettings.DONT_DISABLE_GROOVY_LOG ))
164 {
165 initGroovyLog();
166 }
167
168 runner.run();
169 System.exit( 0 );
170 }
171 catch( SoapUIException e )
172 {
173 log.error( e.toString() );
174 System.exit( 1 );
175 }
176 catch (Throwable e)
177 {
178 e.printStackTrace();
179 System.exit( 2 );
180 }
181 }
182
183 private static void initGroovyLog()
184 {
185 Logger logger = Logger.getLogger( "groovy.log" );
186
187 ConsoleAppender appender = new ConsoleAppender();
188 appender.setWriter( new OutputStreamWriter( System.out ));
189 appender.setLayout( new PatternLayout( "%d{ABSOLUTE} %-5p [%c{1}] %m%n") );
190 logger.addAppender( appender);
191 }
192
193 public SoapUILoadTestRunner()
194 {
195 SoapUI.setTestMonitor( new TestMonitor() );
196 }
197
198 public void setLoadTest(String loadTest)
199 {
200 this.loadTest = loadTest;
201 }
202
203 public void setOutputFolder(String outputFolder)
204 {
205 this.outputFolder = outputFolder;
206 }
207
208 public void setPrintReport(boolean printReport)
209 {
210 this.printReport = printReport;
211 }
212
213 /***
214 * Sets the host to use by all test-requests, the existing endpoint port and path will be used
215 *
216 * @param host the host to use by all requests
217 */
218
219 public void setHost(String host)
220 {
221 this.host = host;
222 }
223
224 /***
225 * Sets the domain to use for any authentications
226 *
227 * @param domain the domain to use for any authentications
228 */
229
230 public void setDomain(String domain)
231 {
232 log.info( "Setting domain to [" + domain + "]" );
233 this.domain = domain;
234 }
235
236 /***
237 * Sets the password to use for any authentications
238 *
239 * @param domain the password to use for any authentications
240 */
241
242 public void setPassword(String password)
243 {
244 log.info( "Setting password to [" + password + "]" );
245 this.password = password;
246 }
247
248 /***
249 * Sets the WSS password-type to use for any authentications. Setting this will result
250 * in the addition of WS-Security UsernamePassword tokens to any outgoing request containing
251 * the specified username and password.
252 *
253 * @param wssPasswordType the wss-password type to use, either 'Text' or 'Digest'
254 */
255
256 public void setWssPasswordType( String wssPasswordType )
257 {
258 this.wssPasswordType = wssPasswordType;
259 }
260
261 /***
262 * Sets the username to use for any authentications
263 *
264 * @param domain the username to use for any authentications
265 */
266
267 public void setUsername(String username)
268 {
269 log.info( "Setting username to [" + username + "]" );
270 this.username = username;
271 }
272
273 /***
274 * Runs the testcases as configured with setXXX methods
275 *
276 * @throws Exception thrown if any tests fail
277 */
278
279 public void run() throws Exception
280 {
281 if( !new File( projectFile ).exists() )
282 throw new SoapUIException( "soapUI project file [" + projectFile + "] not found" );
283
284 WsdlProject project = new WsdlProject( projectFile, null );
285 int suiteCount = 0;
286
287 for( int c = 0; c < project.getTestSuiteCount(); c++ )
288 {
289 if( testSuite == null ||
290 project.getTestSuiteAt( c ).getName().equalsIgnoreCase( testSuite ))
291 {
292 runSuite( project.getTestSuiteAt( c ));
293 suiteCount++;
294 }
295 }
296
297 if( suiteCount == 0 )
298 {
299 log.warn( "No test-suites matched argument [" + testSuite + "]" );
300 }
301 else if( testCaseCount == 0 )
302 {
303 log.warn( "No test-cases matched argument [" + testCase + "]" );
304 }
305 else if( loadTestCount == 0 )
306 {
307 log.warn( "No load-tests matched argument [" + loadTest + "]" );
308 }
309 else if( !failedTests.isEmpty() )
310 {
311 log.info( failedTests.size() + " load tests failed:" );
312 for( LoadTestRunner loadTestRunner : failedTests )
313 {
314 log.info( loadTestRunner.getLoadTest().getName() + ": " + loadTestRunner.getReason() );
315 }
316
317 throw new SoapUIException( "LoadTests failed" );
318 }
319 }
320
321 /***
322 * Run tests in the specified TestSuite
323 *
324 * @param suite the TestSuite to run
325 */
326
327 public void runSuite(TestSuite suite)
328 {
329 long start = System.currentTimeMillis();
330 for( int c = 0; c < suite.getTestCaseCount(); c++ )
331 {
332 String name = suite.getTestCaseAt( c ).getName();
333 if( testCase == null ||
334 name.equalsIgnoreCase( testCase ))
335 {
336 runTestCase( suite.getTestCaseAt( c ));
337 testCaseCount++;
338 }
339 else
340 log.info( "Skipping testcase [" + name + "], filter is [" + testCase + "]");
341 }
342 log.info( "soapUI suite [" + suite.getName() + "] finished in " + (System.currentTimeMillis()-start) + "ms" );
343 }
344
345 /***
346 * Runs the specified TestCase
347 *
348 * @param testCase the testcase to run
349 */
350
351 private void runTestCase(TestCase testCase)
352 {
353 for( int c = 0; c < testCase.getLoadTestCount(); c++ )
354 {
355 String name = testCase.getLoadTestAt( c ).getName();
356 if( loadTest == null || loadTest.equalsIgnoreCase( name ))
357 {
358 runWsdlLoadTest( (WsdlLoadTest) testCase.getLoadTestAt( c ));
359 loadTestCount++;
360 }
361 }
362 }
363
364 /***
365 * Runs the specified LoadTest
366 *
367 * @param loadTest the loadTest to run
368 */
369
370 private void runWsdlLoadTest(WsdlLoadTest loadTest)
371 {
372 try
373 {
374 log.info( "Running LoadTest [" + loadTest.getName() + "]" );
375 loadTest.addLoadTestRunListener(this);
376 LoadTestRunner runner = loadTest.run();
377
378
379 while (runner.getStatus() == LoadTestRunner.Status.RUNNING )
380 {
381 log.info( "LoadTest [" + loadTest.getName() + "] progress: " + runner.getProgress() + ", " +
382 runner.getRunningThreadCount() );
383 Thread.sleep(1000);
384 }
385
386 log.info( "LoadTest [" + loadTest.getName() + "] finished with status " + runner.getStatus().toString() );
387
388 if( printReport )
389 {
390 log.info( "Exporting log and statistics for LoadTest [" + loadTest.getName() + "]" );
391
392 loadTest.getStatisticsModel().finish();
393
394 exportLog( loadTest );
395 exportStatistics( loadTest );
396 }
397 }
398 catch (Exception e)
399 {
400 e.printStackTrace();
401 log.error( e );
402 }
403 }
404
405 private void exportStatistics(WsdlLoadTest loadTest) throws IOException
406 {
407 ExportStatisticsAction exportStatisticsAction = new ExportStatisticsAction( loadTest.getStatisticsModel() );
408 String statisticsFileName = loadTest.getName() + "-statistics.txt";
409 if( outputFolder != null )
410 {
411 ensureOutputFolder();
412 statisticsFileName = outputFolder + File.separator + statisticsFileName;
413 }
414
415 int cnt = exportStatisticsAction.exportToFile( new File( statisticsFileName ));
416 log.info( "Exported " + cnt + " statistics to [" + statisticsFileName + "]" );
417 }
418
419 private void exportLog(WsdlLoadTest loadTest) throws IOException
420 {
421
422 LoadTestLog loadTestLog = loadTest.getLoadTestLog();
423 ExportLoadTestLogAction exportLoadTestLogAction = new ExportLoadTestLogAction(loadTestLog);
424 String logFileName = loadTest.getName() + "-log.txt";
425 if( outputFolder != null )
426 {
427 ensureOutputFolder();
428 logFileName = outputFolder + File.separator + logFileName;
429 }
430
431 int cnt = exportLoadTestLogAction.exportToFile( new File( logFileName ));
432 log.info( "Exported " + cnt + " log items to [" + logFileName + "]" );
433
434 int errorCnt = 0;
435 for( int c = 0; c < loadTestLog.getSize(); c++ )
436 {
437 LoadTestLogEntry entry = (LoadTestLogEntry) loadTestLog.getElementAt( c );
438
439 if( entry != null && entry.isError() )
440 {
441 String entryFileName = loadTest.getName() + "-error-" + errorCnt++ + "-entry.txt";
442 if( outputFolder != null )
443 {
444 ensureOutputFolder();
445 entryFileName = outputFolder + File.separator + entryFileName;
446 }
447
448 try
449 {
450 entry.exportToFile( entryFileName );
451 }
452 catch (Exception e)
453 {
454 e.printStackTrace();
455 }
456 }
457 }
458 log.info( "Exported " + errorCnt + " error results" );
459 }
460
461 private void ensureOutputFolder()
462 {
463 File folder = new File( outputFolder );
464 if( !folder.exists())
465 folder.mkdirs();
466 }
467
468 /***
469 * Sets the soapUI project file containing the tests to run
470 *
471 * @param projectFile the soapUI project file containing the tests to run
472 */
473
474 public void setProjectFile(String projectFile)
475 {
476 log.info( "setting projectFile to [" + projectFile + "]" );
477 this.projectFile = projectFile;
478 }
479
480 /***
481 * Sets the testcase to run
482 *
483 * @param testCase the testcase to run
484 */
485
486 public void setTestCase(String testCase)
487 {
488 log.info( "setting testCase to [" + testCase + "]" );
489 this.testCase = testCase;
490 }
491
492 /***
493 * Sets the endpoint to use for all test requests
494 *
495 * @param endpoint the endpoint to use for all test requests
496 */
497
498 public void setEndpoint(String endpoint)
499 {
500 log.info( "setting test endpoint to [" + endpoint+ "]" );
501 this.endpoint = endpoint.trim();
502 }
503
504 /***
505 * Sets the TestSuite to run. If not set all TestSuites in the specified project file are run
506 *
507 * @param testSuite the testSuite to run.
508 */
509
510 public void setTestSuite(String testSuite)
511 {
512 log.info( "setting testSuite to [" + testSuite + "]" );
513 this.testSuite = testSuite;
514 }
515
516 public void afterLoadTest(LoadTestRunner loadTestRunner, LoadTestRunContext context)
517 {
518 if( loadTestRunner.getStatus() == LoadTestRunner.Status.FAILED )
519 {
520 failedTests.add( loadTestRunner );
521 }
522 }
523
524 public void afterTestCase(LoadTestRunner loadTestRunner, LoadTestRunContext context, TestRunner testRunner, TestRunContext runContext)
525 {
526 }
527
528 public void afterTestStep(LoadTestRunner loadTestRunner, LoadTestRunContext context, TestRunner testRunner, TestRunContext runContext, TestStepResult testStepResult)
529 {
530 }
531
532 public void beforeLoadTest(LoadTestRunner loadTestRunner, LoadTestRunContext context)
533 {
534 }
535
536 public void beforeTestCase(LoadTestRunner loadTestRunner, LoadTestRunContext context, TestRunner testRunner, TestRunContext runContext)
537 {
538 }
539
540 public void beforeTestStep(LoadTestRunner loadTestRunner, LoadTestRunContext context, TestRunner testRunner, TestRunContext runContext, TestStep testStep)
541 {
542 if( testStep instanceof WsdlTestRequestStep )
543 {
544 WsdlTestRequestStep requestStep = (WsdlTestRequestStep) testStep;
545 if( endpoint != null && endpoint.length() > 0 )
546 {
547 requestStep.getTestRequest().setEndpoint( endpoint );
548 }
549
550 if( host != null && host.length() > 0 )
551 {
552 try
553 {
554 String ep = SoapUITestCaseRunner.replaceHost( requestStep.getTestRequest().getEndpoint(), host );
555 requestStep.getTestRequest().setEndpoint( ep );
556 }
557 catch (Exception e)
558 {
559 log.error( "Failed to set host on endpoint", e );
560 }
561 }
562
563 if( username != null && username.length() > 0 )
564 {
565 requestStep.getTestRequest().setUsername( username );
566 }
567
568 if( password != null && password.length() > 0 )
569 {
570 requestStep.getTestRequest().setPassword( password );
571 }
572
573 if( domain != null && domain.length() > 0 )
574 {
575 requestStep.getTestRequest().setDomain( domain );
576 }
577
578 if( wssPasswordType != null && wssPasswordType.length() > 0 )
579 {
580 requestStep.getTestRequest().setWssPasswordType( wssPasswordType.equals( "Digest" ) ?
581 WsdlTestRequest.PW_TYPE_DIGEST : WsdlTestRequest.PW_TYPE_TEXT );
582 }
583 }
584 }
585
586 public void loadTestStarted(LoadTestRunner loadTestRunner, LoadTestRunContext context)
587 {
588 }
589
590 public void loadTestStopped(LoadTestRunner loadTestRunner, LoadTestRunContext context)
591 {
592 }
593 }