1
2
3
4
5
6
7
8
9
10
11
12
13 package com.eviware.soapui.tools;
14
15 import java.io.File;
16 import java.io.IOException;
17 import java.io.OutputStreamWriter;
18 import java.util.ArrayList;
19 import java.util.List;
20
21 import org.apache.commons.cli.CommandLine;
22 import org.apache.commons.cli.CommandLineParser;
23 import org.apache.commons.cli.HelpFormatter;
24 import org.apache.commons.cli.Options;
25 import org.apache.commons.cli.PosixParser;
26 import org.apache.log4j.ConsoleAppender;
27 import org.apache.log4j.Logger;
28 import org.apache.log4j.PatternLayout;
29
30 import com.eviware.soapui.SoapUI;
31 import com.eviware.soapui.impl.wsdl.WsdlProject;
32 import com.eviware.soapui.impl.wsdl.loadtest.WsdlLoadTest;
33 import com.eviware.soapui.impl.wsdl.loadtest.data.actions.ExportLoadTestLogAction;
34 import com.eviware.soapui.impl.wsdl.loadtest.data.actions.ExportStatisticsAction;
35 import com.eviware.soapui.impl.wsdl.loadtest.log.LoadTestLog;
36 import com.eviware.soapui.impl.wsdl.loadtest.log.LoadTestLogEntry;
37 import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestRequest;
38 import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestRequestStep;
39 import com.eviware.soapui.model.testsuite.LoadTestRunContext;
40 import com.eviware.soapui.model.testsuite.LoadTestRunListener;
41 import com.eviware.soapui.model.testsuite.LoadTestRunner;
42 import com.eviware.soapui.model.testsuite.TestCase;
43 import com.eviware.soapui.model.testsuite.TestRunContext;
44 import com.eviware.soapui.model.testsuite.TestRunner;
45 import com.eviware.soapui.model.testsuite.TestStep;
46 import com.eviware.soapui.model.testsuite.TestStepResult;
47 import com.eviware.soapui.model.testsuite.TestSuite;
48 import com.eviware.soapui.monitor.TestMonitor;
49 import com.eviware.soapui.settings.UISettings;
50 import com.eviware.soapui.support.SoapUIException;
51
52 /***
53 * Standalone test-runner used from maven-plugin, can also be used from command-line (see xdocs) or
54 * directly from other classes.
55 * <p>
56 * For standalone usage, set the project file (with setProjectFile) and other desired properties before
57 * calling run</p>
58 *
59 * @author Ole.Matzura
60 */
61
62 public class SoapUILoadTestRunner implements LoadTestRunListener
63 {
64 private String projectFile;
65 private String testSuite;
66 private String testCase;
67 private String endpoint;
68 private final static Logger log = Logger.getLogger( SoapUILoadTestRunner.class );
69 private String domain;
70 private String password;
71 private String username;
72 private String host;
73 private String loadTest;
74 private boolean printReport;
75 private String outputFolder;
76 private List<LoadTestRunner> failedTests = new ArrayList<LoadTestRunner>();
77 private int testCaseCount;
78 private int loadTestCount;
79 private String wssPasswordType;
80
81 public static String TITLE = "soapUI " + SoapUI.SOAPUI_VERSION + " LoadTest Runner";
82
83 /***
84 * Runs the loadtests in the specified soapUI project file, see soapUI xdocs for details.
85 *
86 * @param args
87 * @throws Exception
88 */
89
90 public static void main( String [] args) throws Exception
91 {
92 System.out.println( TITLE );
93 SoapUI.initSoapUILog();
94 SoapUILoadTestRunner runner = new SoapUILoadTestRunner();
95
96 Options options = new Options();
97 options.addOption( "e", true, "Sets the endpoint" );
98 options.addOption( "s", true, "Sets the testsuite" );
99 options.addOption( "c", true, "Sets the testcase" );
100 options.addOption( "l", true, "Sets the loadtest" );
101 options.addOption( "u", true, "Sets the username" );
102 options.addOption( "p", true, "Sets the password" );
103 options.addOption( "w", true, "Sets the WSS password type, either 'Text' or 'Digest'" );
104 options.addOption( "d", true, "Sets the domain" );
105 options.addOption( "h", true, "Sets the host" );
106 options.addOption( "r", false, "Exports statistics and testlogs for each loadtest run" );
107 options.addOption( "f", true, "Sets the output folder to export to" );
108 options.addOption( "t", true, "Sets the soapui-settings.xml file to use" );
109
110 CommandLineParser parser = new PosixParser();
111 CommandLine cmd = parser.parse( options, args);
112
113 String[] args2 = cmd.getArgs();
114 if( args2.length != 1 )
115 {
116 HelpFormatter formatter = new HelpFormatter();
117 formatter.printHelp( "loadtestrunner [options] <soapui-project-file>", options );
118
119 System.err.println( "Missing soapUI project file.." );
120 return;
121 }
122
123 runner.setProjectFile( args2[0] );
124
125 if( cmd.hasOption( "e"))
126 runner.setEndpoint( cmd.getOptionValue( "e" ) );
127
128 if( cmd.hasOption( "s"))
129 runner.setTestSuite( cmd.getOptionValue( "s") );
130
131 if( cmd.hasOption( "c"))
132 runner.setTestCase( cmd.getOptionValue( "c") );
133
134 if( cmd.hasOption( "l"))
135 runner.setLoadTest( cmd.getOptionValue( "l") );
136
137 if( cmd.hasOption( "u"))
138 runner.setUsername( cmd.getOptionValue( "u") );
139
140 if( cmd.hasOption( "p"))
141 runner.setPassword( cmd.getOptionValue( "p") );
142
143 if( cmd.hasOption( "w"))
144 runner.setWssPasswordType( cmd.getOptionValue( "w") );
145
146 if( cmd.hasOption( "d"))
147 runner.setDomain( cmd.getOptionValue( "d") );
148
149 if( cmd.hasOption( "h"))
150 runner.setHost( cmd.getOptionValue( "h") );
151
152 if( cmd.hasOption( "f"))
153 runner.setOutputFolder( cmd.getOptionValue( "f") );
154
155 if( cmd.hasOption( "t"))
156 SoapUI.initSettings( cmd.getOptionValue( "t" ));
157
158 runner.setPrintReport( cmd.hasOption( "r") );
159
160 try
161 {
162 SoapUI.loadExtLibs();
163
164 if( SoapUI.getSettings().getBoolean( UISettings.DONT_DISABLE_GROOVY_LOG ))
165 {
166 initGroovyLog();
167 }
168
169 runner.run();
170 System.exit( 0 );
171 }
172 catch( SoapUIException e )
173 {
174 log.error( e.toString() );
175 System.exit( 1 );
176 }
177 catch (Throwable e)
178 {
179 e.printStackTrace();
180 System.exit( 2 );
181 }
182 }
183
184 private static void initGroovyLog()
185 {
186 Logger logger = Logger.getLogger( "groovy.log" );
187
188 ConsoleAppender appender = new ConsoleAppender();
189 appender.setWriter( new OutputStreamWriter( System.out ));
190 appender.setLayout( new PatternLayout( "%d{ABSOLUTE} %-5p [%c{1}] %m%n") );
191 logger.addAppender( appender);
192 }
193
194 public SoapUILoadTestRunner()
195 {
196 SoapUI.setTestMonitor( new TestMonitor() );
197 }
198
199 public void setLoadTest(String loadTest)
200 {
201 this.loadTest = loadTest;
202 }
203
204 public void setOutputFolder(String outputFolder)
205 {
206 this.outputFolder = outputFolder;
207 }
208
209 public void setPrintReport(boolean printReport)
210 {
211 this.printReport = printReport;
212 }
213
214 /***
215 * Sets the host to use by all test-requests, the existing endpoint port and path will be used
216 *
217 * @param host the host to use by all requests
218 */
219
220 public void setHost(String host)
221 {
222 this.host = host;
223 }
224
225 /***
226 * Sets the domain to use for any authentications
227 *
228 * @param domain the domain to use for any authentications
229 */
230
231 public void setDomain(String domain)
232 {
233 log.info( "Setting domain to [" + domain + "]" );
234 this.domain = domain;
235 }
236
237 /***
238 * Sets the password to use for any authentications
239 *
240 * @param domain the password to use for any authentications
241 */
242
243 public void setPassword(String password)
244 {
245 log.info( "Setting password to [" + password + "]" );
246 this.password = password;
247 }
248
249 /***
250 * Sets the WSS password-type to use for any authentications. Setting this will result
251 * in the addition of WS-Security UsernamePassword tokens to any outgoing request containing
252 * the specified username and password.
253 *
254 * @param wssPasswordType the wss-password type to use, either 'Text' or 'Digest'
255 */
256
257 public void setWssPasswordType( String wssPasswordType )
258 {
259 this.wssPasswordType = wssPasswordType;
260 }
261
262 /***
263 * Sets the username to use for any authentications
264 *
265 * @param domain the username to use for any authentications
266 */
267
268 public void setUsername(String username)
269 {
270 log.info( "Setting username to [" + username + "]" );
271 this.username = username;
272 }
273
274 /***
275 * Runs the testcases as configured with setXXX methods
276 *
277 * @throws Exception thrown if any tests fail
278 */
279
280 public void run() throws Exception
281 {
282 if( !new File( projectFile ).exists() )
283 throw new SoapUIException( "soapUI project file [" + projectFile + "] not found" );
284
285 WsdlProject project = new WsdlProject( projectFile, null );
286 int suiteCount = 0;
287
288 for( int c = 0; c < project.getTestSuiteCount(); c++ )
289 {
290 if( testSuite == null ||
291 project.getTestSuiteAt( c ).getName().equalsIgnoreCase( testSuite ))
292 {
293 runSuite( project.getTestSuiteAt( c ));
294 suiteCount++;
295 }
296 }
297
298 if( suiteCount == 0 )
299 {
300 log.warn( "No test-suites matched argument [" + testSuite + "]" );
301 }
302 else if( testCaseCount == 0 )
303 {
304 log.warn( "No test-cases matched argument [" + testCase + "]" );
305 }
306 else if( loadTestCount == 0 )
307 {
308 log.warn( "No load-tests matched argument [" + loadTest + "]" );
309 }
310 else if( !failedTests.isEmpty() )
311 {
312 log.info( failedTests.size() + " load tests failed:" );
313 for( LoadTestRunner loadTestRunner : failedTests )
314 {
315 log.info( loadTestRunner.getLoadTest().getName() + ": " + loadTestRunner.getReason() );
316 }
317
318 throw new SoapUIException( "LoadTests failed" );
319 }
320 }
321
322 /***
323 * Run tests in the specified TestSuite
324 *
325 * @param suite the TestSuite to run
326 */
327
328 public void runSuite(TestSuite suite)
329 {
330 long start = System.currentTimeMillis();
331 for( int c = 0; c < suite.getTestCaseCount(); c++ )
332 {
333 String name = suite.getTestCaseAt( c ).getName();
334 if( testCase == null ||
335 name.equalsIgnoreCase( testCase ))
336 {
337 runTestCase( suite.getTestCaseAt( c ));
338 testCaseCount++;
339 }
340 else
341 log.info( "Skipping testcase [" + name + "], filter is [" + testCase + "]");
342 }
343 log.info( "soapUI suite [" + suite.getName() + "] finished in " + (System.currentTimeMillis()-start) + "ms" );
344 }
345
346 /***
347 * Runs the specified TestCase
348 *
349 * @param testCase the testcase to run
350 */
351
352 private void runTestCase(TestCase testCase)
353 {
354 for( int c = 0; c < testCase.getLoadTestCount(); c++ )
355 {
356 String name = testCase.getLoadTestAt( c ).getName();
357 if( loadTest == null || loadTest.equalsIgnoreCase( name ))
358 {
359 runWsdlLoadTest( (WsdlLoadTest) testCase.getLoadTestAt( c ));
360 loadTestCount++;
361 }
362 }
363 }
364
365 /***
366 * Runs the specified LoadTest
367 *
368 * @param loadTest the loadTest to run
369 */
370
371 private void runWsdlLoadTest(WsdlLoadTest loadTest)
372 {
373 try
374 {
375 log.info( "Running LoadTest [" + loadTest.getName() + "]" );
376 loadTest.addLoadTestRunListener(this);
377 LoadTestRunner runner = loadTest.run();
378
379
380 while (runner.getStatus() == LoadTestRunner.Status.RUNNING )
381 {
382 log.info( "LoadTest [" + loadTest.getName() + "] progress: " + runner.getProgress() + ", " +
383 runner.getRunningThreadCount() );
384 Thread.sleep(1000);
385 }
386
387 log.info( "LoadTest [" + loadTest.getName() + "] finished with status " + runner.getStatus().toString() );
388
389 if( printReport )
390 {
391 log.info( "Exporting log and statistics for LoadTest [" + loadTest.getName() + "]" );
392
393 loadTest.getStatisticsModel().finish();
394
395 exportLog( loadTest );
396 exportStatistics( loadTest );
397 }
398 }
399 catch (Exception e)
400 {
401 e.printStackTrace();
402 log.error( e );
403 }
404 }
405
406 private void exportStatistics(WsdlLoadTest loadTest) throws IOException
407 {
408 ExportStatisticsAction exportStatisticsAction = new ExportStatisticsAction( loadTest.getStatisticsModel() );
409 String statisticsFileName = loadTest.getName() + "-statistics.txt";
410 if( outputFolder != null )
411 {
412 ensureOutputFolder();
413 statisticsFileName = outputFolder + File.separator + statisticsFileName;
414 }
415
416 int cnt = exportStatisticsAction.exportToFile( new File( statisticsFileName ));
417 log.info( "Exported " + cnt + " statistics to [" + statisticsFileName + "]" );
418 }
419
420 private void exportLog(WsdlLoadTest loadTest) throws IOException
421 {
422
423 LoadTestLog loadTestLog = loadTest.getLoadTestLog();
424 ExportLoadTestLogAction exportLoadTestLogAction = new ExportLoadTestLogAction(loadTestLog);
425 String logFileName = loadTest.getName() + "-log.txt";
426 if( outputFolder != null )
427 {
428 ensureOutputFolder();
429 logFileName = outputFolder + File.separator + logFileName;
430 }
431
432 int cnt = exportLoadTestLogAction.exportToFile( new File( logFileName ));
433 log.info( "Exported " + cnt + " log items to [" + logFileName + "]" );
434
435 int errorCnt = 0;
436 for( int c = 0; c < loadTestLog.getSize(); c++ )
437 {
438 LoadTestLogEntry entry = (LoadTestLogEntry) loadTestLog.getElementAt( c );
439
440 if( entry != null && entry.isError() )
441 {
442 String entryFileName = loadTest.getName() + "-error-" + errorCnt++ + "-entry.txt";
443 if( outputFolder != null )
444 {
445 ensureOutputFolder();
446 entryFileName = outputFolder + File.separator + entryFileName;
447 }
448
449 try
450 {
451 entry.exportToFile( entryFileName );
452 }
453 catch (Exception e)
454 {
455 e.printStackTrace();
456 }
457 }
458 }
459 log.info( "Exported " + errorCnt + " error results" );
460 }
461
462 private void ensureOutputFolder()
463 {
464 File folder = new File( outputFolder );
465 if( !folder.exists())
466 folder.mkdirs();
467 }
468
469 /***
470 * Sets the soapUI project file containing the tests to run
471 *
472 * @param projectFile the soapUI project file containing the tests to run
473 */
474
475 public void setProjectFile(String projectFile)
476 {
477 log.info( "setting projectFile to [" + projectFile + "]" );
478 this.projectFile = projectFile;
479 }
480
481 /***
482 * Sets the testcase to run
483 *
484 * @param testCase the testcase to run
485 */
486
487 public void setTestCase(String testCase)
488 {
489 log.info( "setting testCase to [" + testCase + "]" );
490 this.testCase = testCase;
491 }
492
493 /***
494 * Sets the endpoint to use for all test requests
495 *
496 * @param endpoint the endpoint to use for all test requests
497 */
498
499 public void setEndpoint(String endpoint)
500 {
501 log.info( "setting test endpoint to [" + endpoint+ "]" );
502 this.endpoint = endpoint.trim();
503 }
504
505 /***
506 * Sets the TestSuite to run. If not set all TestSuites in the specified project file are run
507 *
508 * @param testSuite the testSuite to run.
509 */
510
511 public void setTestSuite(String testSuite)
512 {
513 log.info( "setting testSuite to [" + testSuite + "]" );
514 this.testSuite = testSuite;
515 }
516
517 public void afterLoadTest(LoadTestRunner loadTestRunner, LoadTestRunContext context)
518 {
519 if( loadTestRunner.getStatus() == LoadTestRunner.Status.FAILED )
520 {
521 failedTests.add( loadTestRunner );
522 }
523 }
524
525 public void afterTestCase(LoadTestRunner loadTestRunner, LoadTestRunContext context, TestRunner testRunner, TestRunContext runContext)
526 {
527 }
528
529 public void afterTestStep(LoadTestRunner loadTestRunner, LoadTestRunContext context, TestRunner testRunner, TestRunContext runContext, TestStepResult testStepResult)
530 {
531 }
532
533 public void beforeLoadTest(LoadTestRunner loadTestRunner, LoadTestRunContext context)
534 {
535 }
536
537 public void beforeTestCase(LoadTestRunner loadTestRunner, LoadTestRunContext context, TestRunner testRunner, TestRunContext runContext)
538 {
539 }
540
541 public void beforeTestStep(LoadTestRunner loadTestRunner, LoadTestRunContext context, TestRunner testRunner, TestRunContext runContext, TestStep testStep)
542 {
543 if( testStep instanceof WsdlTestRequestStep )
544 {
545 WsdlTestRequestStep requestStep = (WsdlTestRequestStep) testStep;
546 if( endpoint != null && endpoint.length() > 0 )
547 {
548 requestStep.getTestRequest().setEndpoint( endpoint );
549 }
550
551 if( host != null && host.length() > 0 )
552 {
553 try
554 {
555 String ep = SoapUITestCaseRunner.replaceHost( requestStep.getTestRequest().getEndpoint(), host );
556 requestStep.getTestRequest().setEndpoint( ep );
557 }
558 catch (Exception e)
559 {
560 log.error( "Failed to set host on endpoint", e );
561 }
562 }
563
564 if( username != null && username.length() > 0 )
565 {
566 requestStep.getTestRequest().setUsername( username );
567 }
568
569 if( password != null && password.length() > 0 )
570 {
571 requestStep.getTestRequest().setPassword( password );
572 }
573
574 if( domain != null && domain.length() > 0 )
575 {
576 requestStep.getTestRequest().setDomain( domain );
577 }
578
579 if( wssPasswordType != null && wssPasswordType.length() > 0 )
580 {
581 requestStep.getTestRequest().setWssPasswordType( wssPasswordType.equals( "Digest" ) ?
582 WsdlTestRequest.PW_TYPE_DIGEST : WsdlTestRequest.PW_TYPE_TEXT );
583 }
584 }
585 }
586
587 public void loadTestStarted(LoadTestRunner loadTestRunner, LoadTestRunContext context)
588 {
589 }
590
591 public void loadTestStopped(LoadTestRunner loadTestRunner, LoadTestRunContext context)
592 {
593 }
594 }