1
2
3
4
5
6
7
8
9
10
11
12
13 package com.eviware.soapui.tools;
14
15 import java.io.File;
16 import java.io.IOException;
17 import java.util.ArrayList;
18 import java.util.List;
19
20 import org.apache.commons.cli.CommandLine;
21
22 import com.eviware.soapui.SoapUI;
23 import com.eviware.soapui.impl.wsdl.WsdlProject;
24 import com.eviware.soapui.impl.wsdl.loadtest.WsdlLoadTest;
25 import com.eviware.soapui.impl.wsdl.loadtest.data.actions.ExportLoadTestLogAction;
26 import com.eviware.soapui.impl.wsdl.loadtest.data.actions.ExportStatisticsAction;
27 import com.eviware.soapui.impl.wsdl.loadtest.log.LoadTestLog;
28 import com.eviware.soapui.impl.wsdl.loadtest.log.LoadTestLogEntry;
29 import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestRequest;
30 import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestRequestStep;
31 import com.eviware.soapui.model.testsuite.LoadTestRunContext;
32 import com.eviware.soapui.model.testsuite.LoadTestRunListener;
33 import com.eviware.soapui.model.testsuite.LoadTestRunner;
34 import com.eviware.soapui.model.testsuite.TestCase;
35 import com.eviware.soapui.model.testsuite.TestRunContext;
36 import com.eviware.soapui.model.testsuite.TestRunner;
37 import com.eviware.soapui.model.testsuite.TestStep;
38 import com.eviware.soapui.model.testsuite.TestStepResult;
39 import com.eviware.soapui.model.testsuite.TestSuite;
40 import com.eviware.soapui.settings.UISettings;
41 import com.eviware.soapui.support.SoapUIException;
42 import com.eviware.soapui.support.Tools;
43
44 /***
45 * Standalone test-runner used from maven-plugin, can also be used from command-line (see xdocs) or
46 * directly from other classes.
47 * <p>
48 * For standalone usage, set the project file (with setProjectFile) and other desired properties before
49 * calling run</p>
50 *
51 * @author Ole.Matzura
52 */
53
54 public class SoapUILoadTestRunner extends AbstractSoapUIRunner implements LoadTestRunListener
55 {
56 private String testSuite;
57 private String testCase;
58 private String endpoint;
59 private String domain;
60 private String password;
61 private String username;
62 private String host;
63 private String loadTest;
64 private boolean printReport;
65 private String outputFolder;
66 private List<LoadTestRunner> failedTests = new ArrayList<LoadTestRunner>();
67 private int testCaseCount;
68 private int loadTestCount;
69 private int limit = -1;
70 private String wssPasswordType;
71
72 public static String TITLE = "soapUI " + SoapUI.SOAPUI_VERSION + " LoadTest Runner";
73
74 /***
75 * Runs the loadtests in the specified soapUI project file, see soapUI xdocs for details.
76 *
77 * @param args
78 * @throws Exception
79 */
80
81 public static void main( String [] args )
82 {
83 new SoapUILoadTestRunner().runFromCommandLine( args );
84 }
85
86 protected boolean processCommandLine( CommandLine cmd )
87 {
88 if( cmd.hasOption( "e"))
89 setEndpoint( cmd.getOptionValue( "e" ) );
90
91 if( cmd.hasOption( "s"))
92 setTestSuite( cmd.getOptionValue( "s") );
93
94 if( cmd.hasOption( "c"))
95 setTestCase( cmd.getOptionValue( "c") );
96
97 if( cmd.hasOption( "l"))
98 setLoadTest( cmd.getOptionValue( "l") );
99
100 if( cmd.hasOption( "u"))
101 setUsername( cmd.getOptionValue( "u") );
102
103 if( cmd.hasOption( "p"))
104 setPassword( cmd.getOptionValue( "p") );
105
106 if( cmd.hasOption( "w"))
107 setWssPasswordType( cmd.getOptionValue( "w") );
108
109 if( cmd.hasOption( "d"))
110 setDomain( cmd.getOptionValue( "d") );
111
112 if( cmd.hasOption( "h"))
113 setHost( cmd.getOptionValue( "h") );
114
115 if( cmd.hasOption( "m"))
116 setLimit( Integer.parseInt( cmd.getOptionValue( "m") ));
117
118 if( cmd.hasOption( "f"))
119 setOutputFolder( cmd.getOptionValue( "f") );
120
121 if( cmd.hasOption( "t"))
122 setSettingsFile( cmd.getOptionValue( "t" ));
123
124 setPrintReport( cmd.hasOption( "r") );
125
126 return true;
127 }
128
129 public void setLimit( int limit )
130 {
131 this.limit = limit;
132 }
133
134 protected SoapUIOptions initCommandLineOptions()
135 {
136 SoapUIOptions options = new SoapUIOptions( "loadtestrunner" );
137 options.addOption( "e", true, "Sets the endpoint" );
138 options.addOption( "s", true, "Sets the testsuite" );
139 options.addOption( "c", true, "Sets the testcase" );
140 options.addOption( "l", true, "Sets the loadtest" );
141 options.addOption( "u", true, "Sets the username" );
142 options.addOption( "p", true, "Sets the password" );
143 options.addOption( "w", true, "Sets the WSS password type, either 'Text' or 'Digest'" );
144 options.addOption( "d", true, "Sets the domain" );
145 options.addOption( "h", true, "Sets the host" );
146 options.addOption( "m", true, "Overrides the LoadTest limit" );
147 options.addOption( "r", false, "Exports statistics and testlogs for each LoadTest run" );
148 options.addOption( "f", true, "Sets the output folder to export to" );
149 options.addOption( "t", true, "Sets the soapui-settings.xml file to use" );
150 return options;
151 }
152
153 public SoapUILoadTestRunner()
154 {
155 this( TITLE );
156 }
157
158 public SoapUILoadTestRunner( String title )
159 {
160 super( title );
161 }
162
163 public void setLoadTest(String loadTest)
164 {
165 this.loadTest = loadTest;
166 }
167
168 public void setOutputFolder(String outputFolder)
169 {
170 this.outputFolder = outputFolder;
171 }
172
173 public void setPrintReport(boolean printReport)
174 {
175 this.printReport = printReport;
176 }
177
178 /***
179 * Sets the host to use by all test-requests, the existing endpoint port and path will be used
180 *
181 * @param host the host to use by all requests
182 */
183
184 public void setHost(String host)
185 {
186 this.host = host;
187 }
188
189 /***
190 * Sets the domain to use for any authentications
191 *
192 * @param domain the domain to use for any authentications
193 */
194
195 public void setDomain(String domain)
196 {
197 this.domain = domain;
198 }
199
200 /***
201 * Sets the password to use for any authentications
202 *
203 * @param domain the password to use for any authentications
204 */
205
206 public void setPassword(String password)
207 {
208 this.password = password;
209 }
210
211 /***
212 * Sets the WSS password-type to use for any authentications. Setting this will result
213 * in the addition of WS-Security UsernamePassword tokens to any outgoing request containing
214 * the specified username and password.
215 *
216 * @param wssPasswordType the wss-password type to use, either 'Text' or 'Digest'
217 */
218
219 public void setWssPasswordType( String wssPasswordType )
220 {
221 this.wssPasswordType = wssPasswordType;
222 }
223
224 /***
225 * Sets the username to use for any authentications
226 *
227 * @param domain the username to use for any authentications
228 */
229
230 public void setUsername(String username)
231 {
232 this.username = username;
233 }
234
235 /***
236 * Runs the testcases as configured with setXXX methods
237 *
238 * @throws Exception thrown if any tests fail
239 */
240
241 public boolean runRunner() throws Exception
242 {
243 if( SoapUI.getSettings().getBoolean( UISettings.DONT_DISABLE_GROOVY_LOG ))
244 {
245 initGroovyLog();
246 }
247
248 String projectFile = getProjectFile();
249
250 WsdlProject project = new WsdlProject( projectFile );
251 if( project.isDisabled() )
252 throw new Exception( "Failed to load soapUI project file [" + projectFile + "]" );
253 int suiteCount = 0;
254
255 for( int c = 0; c < project.getTestSuiteCount(); c++ )
256 {
257 if( testSuite == null ||
258 project.getTestSuiteAt( c ).getName().equalsIgnoreCase( testSuite ))
259 {
260 runSuite( project.getTestSuiteAt( c ));
261 suiteCount++;
262 }
263 }
264
265 if( suiteCount == 0 )
266 {
267 log.warn( "No test-suites matched argument [" + testSuite + "]" );
268 }
269 else if( testCaseCount == 0 )
270 {
271 log.warn( "No test-cases matched argument [" + testCase + "]" );
272 }
273 else if( loadTestCount == 0 )
274 {
275 log.warn( "No load-tests matched argument [" + loadTest + "]" );
276 }
277 else if( !failedTests.isEmpty() )
278 {
279 log.info( failedTests.size() + " load tests failed:" );
280 for( LoadTestRunner loadTestRunner : failedTests )
281 {
282 log.info( loadTestRunner.getLoadTest().getName() + ": " + loadTestRunner.getReason() );
283 }
284
285 throw new SoapUIException( "LoadTests failed" );
286 }
287
288 return true;
289 }
290
291 /***
292 * Run tests in the specified TestSuite
293 *
294 * @param suite the TestSuite to run
295 */
296
297 public void runSuite(TestSuite suite)
298 {
299 long start = System.currentTimeMillis();
300 for( int c = 0; c < suite.getTestCaseCount(); c++ )
301 {
302 String name = suite.getTestCaseAt( c ).getName();
303 if( testCase == null ||
304 name.equalsIgnoreCase( testCase ))
305 {
306 runTestCase( suite.getTestCaseAt( c ));
307 testCaseCount++;
308 }
309 else
310 log.info( "Skipping testcase [" + name + "], filter is [" + testCase + "]");
311 }
312 log.info( "soapUI suite [" + suite.getName() + "] finished in " + (System.currentTimeMillis()-start) + "ms" );
313 }
314
315 /***
316 * Runs the specified TestCase
317 *
318 * @param testCase the testcase to run
319 */
320
321 private void runTestCase(TestCase testCase)
322 {
323 for( int c = 0; c < testCase.getLoadTestCount(); c++ )
324 {
325 String name = testCase.getLoadTestAt( c ).getName();
326 if( loadTest == null || loadTest.equalsIgnoreCase( name ))
327 {
328 runWsdlLoadTest( (WsdlLoadTest) testCase.getLoadTestAt( c ));
329 loadTestCount++;
330 }
331 }
332 }
333
334 /***
335 * Runs the specified LoadTest
336 *
337 * @param loadTest the loadTest to run
338 */
339
340 private void runWsdlLoadTest(WsdlLoadTest loadTest)
341 {
342 try
343 {
344 log.info( "Running LoadTest [" + loadTest.getName() + "]" );
345 if( limit >= 0 )
346 {
347 log.info( "Overriding limit [" + loadTest.getTestLimit() + "] with specified [" + limit + "]" );
348 loadTest.setTestLimit( limit );
349 }
350
351 loadTest.addLoadTestRunListener(this);
352 LoadTestRunner runner = loadTest.run();
353
354
355 while (runner.getStatus() == LoadTestRunner.Status.RUNNING )
356 {
357 log.info( "LoadTest [" + loadTest.getName() + "] progress: " + runner.getProgress() + ", " +
358 runner.getRunningThreadCount() );
359 Thread.sleep(1000);
360 }
361
362 log.info( "LoadTest [" + loadTest.getName() + "] finished with status " + runner.getStatus().toString() );
363
364 if( printReport )
365 {
366 log.info( "Exporting log and statistics for LoadTest [" + loadTest.getName() + "]" );
367
368 loadTest.getStatisticsModel().finish();
369
370 exportLog( loadTest );
371 exportStatistics( loadTest );
372 }
373 }
374 catch (Exception e)
375 {
376 SoapUI.logError( e );
377 log.error( e );
378 }
379 }
380
381 private void exportStatistics(WsdlLoadTest loadTest) throws IOException
382 {
383 ExportStatisticsAction exportStatisticsAction = new ExportStatisticsAction( loadTest.getStatisticsModel() );
384 String statisticsFileName = loadTest.getName() + "-statistics.txt";
385 if( outputFolder != null )
386 {
387 ensureOutputFolder();
388 statisticsFileName = outputFolder + File.separator + statisticsFileName;
389 }
390
391 int cnt = exportStatisticsAction.exportToFile( new File( statisticsFileName ));
392 log.info( "Exported " + cnt + " statistics to [" + statisticsFileName + "]" );
393 }
394
395 private void exportLog(WsdlLoadTest loadTest) throws IOException
396 {
397
398 LoadTestLog loadTestLog = loadTest.getLoadTestLog();
399 ExportLoadTestLogAction exportLoadTestLogAction = new ExportLoadTestLogAction(loadTestLog, null);
400 String logFileName = loadTest.getName() + "-log.txt";
401 if( outputFolder != null )
402 {
403 ensureOutputFolder();
404 logFileName = outputFolder + File.separator + logFileName;
405 }
406
407 int cnt = exportLoadTestLogAction.exportToFile( new File( logFileName ));
408 log.info( "Exported " + cnt + " log items to [" + logFileName + "]" );
409
410 int errorCnt = 0;
411 for( int c = 0; c < loadTestLog.getSize(); c++ )
412 {
413 LoadTestLogEntry entry = (LoadTestLogEntry) loadTestLog.getElementAt( c );
414
415 if( entry != null && entry.isError() )
416 {
417 String entryFileName = loadTest.getName() + "-error-" + errorCnt++ + "-entry.txt";
418 if( outputFolder != null )
419 {
420 ensureOutputFolder();
421 entryFileName = outputFolder + File.separator + entryFileName;
422 }
423
424 try
425 {
426 entry.exportToFile( entryFileName );
427 }
428 catch (Exception e)
429 {
430 SoapUI.logError( e );
431 }
432 }
433 }
434 log.info( "Exported " + errorCnt + " error results" );
435 }
436
437 private void ensureOutputFolder()
438 {
439 File folder = new File( outputFolder );
440 if( !folder.exists())
441 folder.mkdirs();
442 }
443
444 /***
445 * Sets the testcase to run
446 *
447 * @param testCase the testcase to run
448 */
449
450 public void setTestCase(String testCase)
451 {
452 this.testCase = testCase;
453 }
454
455 /***
456 * Sets the endpoint to use for all test requests
457 *
458 * @param endpoint the endpoint to use for all test requests
459 */
460
461 public void setEndpoint(String endpoint)
462 {
463 this.endpoint = endpoint.trim();
464 }
465
466 /***
467 * Sets the TestSuite to run. If not set all TestSuites in the specified project file are run
468 *
469 * @param testSuite the testSuite to run.
470 */
471
472 public void setTestSuite(String testSuite)
473 {
474 this.testSuite = testSuite;
475 }
476
477 public void afterLoadTest(LoadTestRunner loadTestRunner, LoadTestRunContext context)
478 {
479 if( loadTestRunner.getStatus() == LoadTestRunner.Status.FAILED )
480 {
481 failedTests.add( loadTestRunner );
482 }
483 }
484
485 public void afterTestCase(LoadTestRunner loadTestRunner, LoadTestRunContext context, TestRunner testRunner, TestRunContext runContext)
486 {
487 }
488
489 public void afterTestStep(LoadTestRunner loadTestRunner, LoadTestRunContext context, TestRunner testRunner, TestRunContext runContext, TestStepResult testStepResult)
490 {
491 }
492
493 public void beforeLoadTest(LoadTestRunner loadTestRunner, LoadTestRunContext context)
494 {
495 }
496
497 public void beforeTestCase(LoadTestRunner loadTestRunner, LoadTestRunContext context, TestRunner testRunner, TestRunContext runContext)
498 {
499 }
500
501 public void beforeTestStep(LoadTestRunner loadTestRunner, LoadTestRunContext context, TestRunner testRunner, TestRunContext runContext, TestStep testStep)
502 {
503 if( testStep instanceof WsdlTestRequestStep )
504 {
505 WsdlTestRequestStep requestStep = (WsdlTestRequestStep) testStep;
506 if( endpoint != null && endpoint.length() > 0 )
507 {
508 requestStep.getTestRequest().setEndpoint( endpoint );
509 }
510
511 if( host != null && host.length() > 0 )
512 {
513 try
514 {
515 String ep = Tools.replaceHost( requestStep.getTestRequest().getEndpoint(), host );
516 requestStep.getTestRequest().setEndpoint( ep );
517 }
518 catch (Exception e)
519 {
520 log.error( "Failed to set host on endpoint", e );
521 }
522 }
523
524 if( username != null && username.length() > 0 )
525 {
526 requestStep.getTestRequest().setUsername( username );
527 }
528
529 if( password != null && password.length() > 0 )
530 {
531 requestStep.getTestRequest().setPassword( password );
532 }
533
534 if( domain != null && domain.length() > 0 )
535 {
536 requestStep.getTestRequest().setDomain( domain );
537 }
538
539 if( wssPasswordType != null && wssPasswordType.length() > 0 )
540 {
541 requestStep.getTestRequest().setWssPasswordType( wssPasswordType.equals( "Digest" ) ?
542 WsdlTestRequest.PW_TYPE_DIGEST : WsdlTestRequest.PW_TYPE_TEXT );
543 }
544 }
545 }
546
547 public void loadTestStarted(LoadTestRunner loadTestRunner, LoadTestRunContext context)
548 {
549 }
550
551 public void loadTestStopped(LoadTestRunner loadTestRunner, LoadTestRunContext context)
552 {
553 }
554 }