1
2
3
4
5
6
7
8
9
10
11
12
13 package com.eviware.soapui.tools;
14
15 import java.io.File;
16 import java.io.IOException;
17 import java.util.ArrayList;
18 import java.util.List;
19
20 import org.apache.commons.cli.CommandLine;
21
22 import com.eviware.soapui.SoapUI;
23 import com.eviware.soapui.impl.wsdl.WsdlProject;
24 import com.eviware.soapui.impl.wsdl.loadtest.WsdlLoadTest;
25 import com.eviware.soapui.impl.wsdl.loadtest.data.actions.ExportLoadTestLogAction;
26 import com.eviware.soapui.impl.wsdl.loadtest.data.actions.ExportStatisticsAction;
27 import com.eviware.soapui.impl.wsdl.loadtest.log.LoadTestLog;
28 import com.eviware.soapui.impl.wsdl.loadtest.log.LoadTestLogEntry;
29 import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestRequest;
30 import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestRequestStep;
31 import com.eviware.soapui.model.testsuite.LoadTestRunContext;
32 import com.eviware.soapui.model.testsuite.LoadTestRunListener;
33 import com.eviware.soapui.model.testsuite.LoadTestRunner;
34 import com.eviware.soapui.model.testsuite.TestCase;
35 import com.eviware.soapui.model.testsuite.TestRunContext;
36 import com.eviware.soapui.model.testsuite.TestRunner;
37 import com.eviware.soapui.model.testsuite.TestStep;
38 import com.eviware.soapui.model.testsuite.TestStepResult;
39 import com.eviware.soapui.model.testsuite.TestSuite;
40 import com.eviware.soapui.settings.UISettings;
41 import com.eviware.soapui.support.SoapUIException;
42 import com.eviware.soapui.support.Tools;
43
44 /***
45 * Standalone test-runner used from maven-plugin, can also be used from command-line (see xdocs) or
46 * directly from other classes.
47 * <p>
48 * For standalone usage, set the project file (with setProjectFile) and other desired properties before
49 * calling run</p>
50 *
51 * @author Ole.Matzura
52 */
53
54 public class SoapUILoadTestRunner extends AbstractSoapUIRunner implements LoadTestRunListener
55 {
56 private String testSuite;
57 private String testCase;
58 private String endpoint;
59 private String domain;
60 private String password;
61 private String username;
62 private String host;
63 private String loadTest;
64 private boolean printReport;
65 private String outputFolder;
66 private List<LoadTestRunner> failedTests = new ArrayList<LoadTestRunner>();
67 private int testCaseCount;
68 private int loadTestCount;
69 private int limit = -1;
70 private String wssPasswordType;
71
72 public static String TITLE = "soapUI " + SoapUI.SOAPUI_VERSION + " LoadTest Runner";
73
74 /***
75 * Runs the loadtests in the specified soapUI project file, see soapUI xdocs for details.
76 *
77 * @param args
78 * @throws Exception
79 */
80
81 public static void main( String [] args )
82 {
83 new SoapUILoadTestRunner().runFromCommandLine( args );
84 }
85
86 protected boolean processCommandLine( CommandLine cmd )
87 {
88 if( cmd.hasOption( "e"))
89 setEndpoint( cmd.getOptionValue( "e" ) );
90
91 if( cmd.hasOption( "s"))
92 setTestSuite( cmd.getOptionValue( "s") );
93
94 if( cmd.hasOption( "c"))
95 setTestCase( cmd.getOptionValue( "c") );
96
97 if( cmd.hasOption( "l"))
98 setLoadTest( cmd.getOptionValue( "l") );
99
100 if( cmd.hasOption( "u"))
101 setUsername( cmd.getOptionValue( "u") );
102
103 if( cmd.hasOption( "p"))
104 setPassword( cmd.getOptionValue( "p") );
105
106 if( cmd.hasOption( "w"))
107 setWssPasswordType( cmd.getOptionValue( "w") );
108
109 if( cmd.hasOption( "d"))
110 setDomain( cmd.getOptionValue( "d") );
111
112 if( cmd.hasOption( "h"))
113 setHost( cmd.getOptionValue( "h") );
114
115 if( cmd.hasOption( "m"))
116 setLimit( Integer.parseInt( cmd.getOptionValue( "m") ));
117
118 if( cmd.hasOption( "f"))
119 setOutputFolder( cmd.getOptionValue( "f") );
120
121 if( cmd.hasOption( "t"))
122 setSettingsFile( cmd.getOptionValue( "t" ));
123
124 setPrintReport( cmd.hasOption( "r") );
125
126 return true;
127 }
128
129 public void setLimit( int limit )
130 {
131 this.limit = limit;
132 }
133
134 protected SoapUIOptions initCommandLineOptions()
135 {
136 SoapUIOptions options = new SoapUIOptions( "loadtestrunner" );
137 options.addOption( "e", true, "Sets the endpoint" );
138 options.addOption( "s", true, "Sets the testsuite" );
139 options.addOption( "c", true, "Sets the testcase" );
140 options.addOption( "l", true, "Sets the loadtest" );
141 options.addOption( "u", true, "Sets the username" );
142 options.addOption( "p", true, "Sets the password" );
143 options.addOption( "w", true, "Sets the WSS password type, either 'Text' or 'Digest'" );
144 options.addOption( "d", true, "Sets the domain" );
145 options.addOption( "h", true, "Sets the host" );
146 options.addOption( "m", true, "Overrides the LoadTest limit" );
147 options.addOption( "r", false, "Exports statistics and testlogs for each LoadTest run" );
148 options.addOption( "f", true, "Sets the output folder to export to" );
149 options.addOption( "t", true, "Sets the soapui-settings.xml file to use" );
150 return options;
151 }
152
153 public SoapUILoadTestRunner()
154 {
155 this( TITLE );
156 }
157
158 public SoapUILoadTestRunner( String title )
159 {
160 super( title );
161 }
162
163 public void setLoadTest(String loadTest)
164 {
165 this.loadTest = loadTest;
166 }
167
168 public void setOutputFolder(String outputFolder)
169 {
170 this.outputFolder = outputFolder;
171 }
172
173 public void setPrintReport(boolean printReport)
174 {
175 this.printReport = printReport;
176 }
177
178 /***
179 * Sets the host to use by all test-requests, the existing endpoint port and path will be used
180 *
181 * @param host the host to use by all requests
182 */
183
184 public void setHost(String host)
185 {
186 this.host = host;
187 }
188
189 /***
190 * Sets the domain to use for any authentications
191 *
192 * @param domain the domain to use for any authentications
193 */
194
195 public void setDomain(String domain)
196 {
197 log.info( "Setting domain to [" + domain + "]" );
198 this.domain = domain;
199 }
200
201 /***
202 * Sets the password to use for any authentications
203 *
204 * @param domain the password to use for any authentications
205 */
206
207 public void setPassword(String password)
208 {
209 log.info( "Setting password to [" + password + "]" );
210 this.password = password;
211 }
212
213 /***
214 * Sets the WSS password-type to use for any authentications. Setting this will result
215 * in the addition of WS-Security UsernamePassword tokens to any outgoing request containing
216 * the specified username and password.
217 *
218 * @param wssPasswordType the wss-password type to use, either 'Text' or 'Digest'
219 */
220
221 public void setWssPasswordType( String wssPasswordType )
222 {
223 this.wssPasswordType = wssPasswordType;
224 }
225
226 /***
227 * Sets the username to use for any authentications
228 *
229 * @param domain the username to use for any authentications
230 */
231
232 public void setUsername(String username)
233 {
234 log.info( "Setting username to [" + username + "]" );
235 this.username = username;
236 }
237
238 /***
239 * Runs the testcases as configured with setXXX methods
240 *
241 * @throws Exception thrown if any tests fail
242 */
243
244 public void runRunner() throws Exception
245 {
246 if( SoapUI.getSettings().getBoolean( UISettings.DONT_DISABLE_GROOVY_LOG ))
247 {
248 initGroovyLog();
249 }
250
251 String projectFile = getProjectFile();
252
253 WsdlProject project = new WsdlProject( projectFile );
254 if( project.isDisabled() )
255 throw new Exception( "Failed to load soapUI project file [" + projectFile + "]" );
256 int suiteCount = 0;
257
258 for( int c = 0; c < project.getTestSuiteCount(); c++ )
259 {
260 if( testSuite == null ||
261 project.getTestSuiteAt( c ).getName().equalsIgnoreCase( testSuite ))
262 {
263 runSuite( project.getTestSuiteAt( c ));
264 suiteCount++;
265 }
266 }
267
268 if( suiteCount == 0 )
269 {
270 log.warn( "No test-suites matched argument [" + testSuite + "]" );
271 }
272 else if( testCaseCount == 0 )
273 {
274 log.warn( "No test-cases matched argument [" + testCase + "]" );
275 }
276 else if( loadTestCount == 0 )
277 {
278 log.warn( "No load-tests matched argument [" + loadTest + "]" );
279 }
280 else if( !failedTests.isEmpty() )
281 {
282 log.info( failedTests.size() + " load tests failed:" );
283 for( LoadTestRunner loadTestRunner : failedTests )
284 {
285 log.info( loadTestRunner.getLoadTest().getName() + ": " + loadTestRunner.getReason() );
286 }
287
288 throw new SoapUIException( "LoadTests failed" );
289 }
290 }
291
292 /***
293 * Run tests in the specified TestSuite
294 *
295 * @param suite the TestSuite to run
296 */
297
298 public void runSuite(TestSuite suite)
299 {
300 long start = System.currentTimeMillis();
301 for( int c = 0; c < suite.getTestCaseCount(); c++ )
302 {
303 String name = suite.getTestCaseAt( c ).getName();
304 if( testCase == null ||
305 name.equalsIgnoreCase( testCase ))
306 {
307 runTestCase( suite.getTestCaseAt( c ));
308 testCaseCount++;
309 }
310 else
311 log.info( "Skipping testcase [" + name + "], filter is [" + testCase + "]");
312 }
313 log.info( "soapUI suite [" + suite.getName() + "] finished in " + (System.currentTimeMillis()-start) + "ms" );
314 }
315
316 /***
317 * Runs the specified TestCase
318 *
319 * @param testCase the testcase to run
320 */
321
322 private void runTestCase(TestCase testCase)
323 {
324 for( int c = 0; c < testCase.getLoadTestCount(); c++ )
325 {
326 String name = testCase.getLoadTestAt( c ).getName();
327 if( loadTest == null || loadTest.equalsIgnoreCase( name ))
328 {
329 runWsdlLoadTest( (WsdlLoadTest) testCase.getLoadTestAt( c ));
330 loadTestCount++;
331 }
332 }
333 }
334
335 /***
336 * Runs the specified LoadTest
337 *
338 * @param loadTest the loadTest to run
339 */
340
341 private void runWsdlLoadTest(WsdlLoadTest loadTest)
342 {
343 try
344 {
345 log.info( "Running LoadTest [" + loadTest.getName() + "]" );
346 if( limit >= 0 )
347 {
348 log.info( "Overriding limit [" + loadTest.getTestLimit() + "] with specified [" + limit + "]" );
349 loadTest.setTestLimit( limit );
350 }
351
352 loadTest.addLoadTestRunListener(this);
353 LoadTestRunner runner = loadTest.run();
354
355
356 while (runner.getStatus() == LoadTestRunner.Status.RUNNING )
357 {
358 log.info( "LoadTest [" + loadTest.getName() + "] progress: " + runner.getProgress() + ", " +
359 runner.getRunningThreadCount() );
360 Thread.sleep(1000);
361 }
362
363 log.info( "LoadTest [" + loadTest.getName() + "] finished with status " + runner.getStatus().toString() );
364
365 if( printReport )
366 {
367 log.info( "Exporting log and statistics for LoadTest [" + loadTest.getName() + "]" );
368
369 loadTest.getStatisticsModel().finish();
370
371 exportLog( loadTest );
372 exportStatistics( loadTest );
373 }
374 }
375 catch (Exception e)
376 {
377 SoapUI.logError( e );
378 log.error( e );
379 }
380 }
381
382 private void exportStatistics(WsdlLoadTest loadTest) throws IOException
383 {
384 ExportStatisticsAction exportStatisticsAction = new ExportStatisticsAction( loadTest.getStatisticsModel() );
385 String statisticsFileName = loadTest.getName() + "-statistics.txt";
386 if( outputFolder != null )
387 {
388 ensureOutputFolder();
389 statisticsFileName = outputFolder + File.separator + statisticsFileName;
390 }
391
392 int cnt = exportStatisticsAction.exportToFile( new File( statisticsFileName ));
393 log.info( "Exported " + cnt + " statistics to [" + statisticsFileName + "]" );
394 }
395
396 private void exportLog(WsdlLoadTest loadTest) throws IOException
397 {
398
399 LoadTestLog loadTestLog = loadTest.getLoadTestLog();
400 ExportLoadTestLogAction exportLoadTestLogAction = new ExportLoadTestLogAction(loadTestLog, null);
401 String logFileName = loadTest.getName() + "-log.txt";
402 if( outputFolder != null )
403 {
404 ensureOutputFolder();
405 logFileName = outputFolder + File.separator + logFileName;
406 }
407
408 int cnt = exportLoadTestLogAction.exportToFile( new File( logFileName ));
409 log.info( "Exported " + cnt + " log items to [" + logFileName + "]" );
410
411 int errorCnt = 0;
412 for( int c = 0; c < loadTestLog.getSize(); c++ )
413 {
414 LoadTestLogEntry entry = (LoadTestLogEntry) loadTestLog.getElementAt( c );
415
416 if( entry != null && entry.isError() )
417 {
418 String entryFileName = loadTest.getName() + "-error-" + errorCnt++ + "-entry.txt";
419 if( outputFolder != null )
420 {
421 ensureOutputFolder();
422 entryFileName = outputFolder + File.separator + entryFileName;
423 }
424
425 try
426 {
427 entry.exportToFile( entryFileName );
428 }
429 catch (Exception e)
430 {
431 SoapUI.logError( e );
432 }
433 }
434 }
435 log.info( "Exported " + errorCnt + " error results" );
436 }
437
438 private void ensureOutputFolder()
439 {
440 File folder = new File( outputFolder );
441 if( !folder.exists())
442 folder.mkdirs();
443 }
444
445 /***
446 * Sets the testcase to run
447 *
448 * @param testCase the testcase to run
449 */
450
451 public void setTestCase(String testCase)
452 {
453 this.testCase = testCase;
454 }
455
456 /***
457 * Sets the endpoint to use for all test requests
458 *
459 * @param endpoint the endpoint to use for all test requests
460 */
461
462 public void setEndpoint(String endpoint)
463 {
464 this.endpoint = endpoint.trim();
465 }
466
467 /***
468 * Sets the TestSuite to run. If not set all TestSuites in the specified project file are run
469 *
470 * @param testSuite the testSuite to run.
471 */
472
473 public void setTestSuite(String testSuite)
474 {
475 this.testSuite = testSuite;
476 }
477
478 public void afterLoadTest(LoadTestRunner loadTestRunner, LoadTestRunContext context)
479 {
480 if( loadTestRunner.getStatus() == LoadTestRunner.Status.FAILED )
481 {
482 failedTests.add( loadTestRunner );
483 }
484 }
485
486 public void afterTestCase(LoadTestRunner loadTestRunner, LoadTestRunContext context, TestRunner testRunner, TestRunContext runContext)
487 {
488 }
489
490 public void afterTestStep(LoadTestRunner loadTestRunner, LoadTestRunContext context, TestRunner testRunner, TestRunContext runContext, TestStepResult testStepResult)
491 {
492 }
493
494 public void beforeLoadTest(LoadTestRunner loadTestRunner, LoadTestRunContext context)
495 {
496 }
497
498 public void beforeTestCase(LoadTestRunner loadTestRunner, LoadTestRunContext context, TestRunner testRunner, TestRunContext runContext)
499 {
500 }
501
502 public void beforeTestStep(LoadTestRunner loadTestRunner, LoadTestRunContext context, TestRunner testRunner, TestRunContext runContext, TestStep testStep)
503 {
504 if( testStep instanceof WsdlTestRequestStep )
505 {
506 WsdlTestRequestStep requestStep = (WsdlTestRequestStep) testStep;
507 if( endpoint != null && endpoint.length() > 0 )
508 {
509 requestStep.getTestRequest().setEndpoint( endpoint );
510 }
511
512 if( host != null && host.length() > 0 )
513 {
514 try
515 {
516 String ep = Tools.replaceHost( requestStep.getTestRequest().getEndpoint(), host );
517 requestStep.getTestRequest().setEndpoint( ep );
518 }
519 catch (Exception e)
520 {
521 log.error( "Failed to set host on endpoint", e );
522 }
523 }
524
525 if( username != null && username.length() > 0 )
526 {
527 requestStep.getTestRequest().setUsername( username );
528 }
529
530 if( password != null && password.length() > 0 )
531 {
532 requestStep.getTestRequest().setPassword( password );
533 }
534
535 if( domain != null && domain.length() > 0 )
536 {
537 requestStep.getTestRequest().setDomain( domain );
538 }
539
540 if( wssPasswordType != null && wssPasswordType.length() > 0 )
541 {
542 requestStep.getTestRequest().setWssPasswordType( wssPasswordType.equals( "Digest" ) ?
543 WsdlTestRequest.PW_TYPE_DIGEST : WsdlTestRequest.PW_TYPE_TEXT );
544 }
545 }
546 }
547
548 public void loadTestStarted(LoadTestRunner loadTestRunner, LoadTestRunContext context)
549 {
550 }
551
552 public void loadTestStopped(LoadTestRunner loadTestRunner, LoadTestRunContext context)
553 {
554 }
555 }