1
2
3
4
5
6
7
8
9
10
11
12
13 package com.eviware.soapui.tools;
14
15 import java.io.File;
16 import java.io.IOException;
17 import java.util.ArrayList;
18 import java.util.List;
19
20 import org.apache.commons.cli.CommandLine;
21
22 import com.eviware.soapui.SoapUI;
23 import com.eviware.soapui.impl.wsdl.WsdlProject;
24 import com.eviware.soapui.impl.wsdl.loadtest.WsdlLoadTest;
25 import com.eviware.soapui.impl.wsdl.loadtest.data.actions.ExportLoadTestLogAction;
26 import com.eviware.soapui.impl.wsdl.loadtest.data.actions.ExportStatisticsAction;
27 import com.eviware.soapui.impl.wsdl.loadtest.log.LoadTestLog;
28 import com.eviware.soapui.impl.wsdl.loadtest.log.LoadTestLogEntry;
29 import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestRequest;
30 import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestRequestStep;
31 import com.eviware.soapui.model.testsuite.LoadTestRunContext;
32 import com.eviware.soapui.model.testsuite.LoadTestRunListener;
33 import com.eviware.soapui.model.testsuite.LoadTestRunner;
34 import com.eviware.soapui.model.testsuite.TestCase;
35 import com.eviware.soapui.model.testsuite.TestRunContext;
36 import com.eviware.soapui.model.testsuite.TestRunner;
37 import com.eviware.soapui.model.testsuite.TestStep;
38 import com.eviware.soapui.model.testsuite.TestStepResult;
39 import com.eviware.soapui.model.testsuite.TestSuite;
40 import com.eviware.soapui.settings.UISettings;
41 import com.eviware.soapui.support.SoapUIException;
42 import com.eviware.soapui.support.Tools;
43
44 /***
45 * Standalone test-runner used from maven-plugin, can also be used from command-line (see xdocs) or
46 * directly from other classes.
47 * <p>
48 * For standalone usage, set the project file (with setProjectFile) and other desired properties before
49 * calling run</p>
50 *
51 * @author Ole.Matzura
52 */
53
54 public class SoapUILoadTestRunner extends AbstractSoapUIRunner implements LoadTestRunListener
55 {
56 private String testSuite;
57 private String testCase;
58 private String endpoint;
59 private String domain;
60 private String password;
61 private String username;
62 private String host;
63 private String loadTest;
64 private boolean printReport;
65 private String outputFolder;
66 private List<LoadTestRunner> failedTests = new ArrayList<LoadTestRunner>();
67 private int testCaseCount;
68 private int loadTestCount;
69 private int limit = -1;
70 private String wssPasswordType;
71 private long threadCount = -1;
72
73 public static String TITLE = "soapUI " + SoapUI.SOAPUI_VERSION + " LoadTest Runner";
74
75 /***
76 * Runs the loadtests in the specified soapUI project file, see soapUI xdocs for details.
77 *
78 * @param args
79 * @throws Exception
80 */
81
82 public static void main( String [] args )
83 {
84 new SoapUILoadTestRunner().runFromCommandLine( args );
85 }
86
87 protected boolean processCommandLine( CommandLine cmd )
88 {
89 if( cmd.hasOption( "e"))
90 setEndpoint( cmd.getOptionValue( "e" ) );
91
92 if( cmd.hasOption( "s"))
93 setTestSuite( getCommandLineOptionSubstSpace( cmd, "s") );
94
95 if( cmd.hasOption( "c"))
96 setTestCase( cmd.getOptionValue( "c") );
97
98 if( cmd.hasOption( "l"))
99 setLoadTest( cmd.getOptionValue( "l") );
100
101 if( cmd.hasOption( "u"))
102 setUsername( cmd.getOptionValue( "u") );
103
104 if( cmd.hasOption( "p"))
105 setPassword( cmd.getOptionValue( "p") );
106
107 if( cmd.hasOption( "w"))
108 setWssPasswordType( cmd.getOptionValue( "w") );
109
110 if( cmd.hasOption( "d"))
111 setDomain( cmd.getOptionValue( "d") );
112
113 if( cmd.hasOption( "h"))
114 setHost( cmd.getOptionValue( "h") );
115
116 if( cmd.hasOption( "m"))
117 setLimit( Integer.parseInt( cmd.getOptionValue( "m") ));
118
119 if( cmd.hasOption( "h"))
120 setThreadCount( Integer.parseInt( cmd.getOptionValue( "h") ));
121
122 if( cmd.hasOption( "f"))
123 setOutputFolder( getCommandLineOptionSubstSpace( cmd, "f") );
124
125 if( cmd.hasOption( "t"))
126 setSettingsFile( getCommandLineOptionSubstSpace( cmd, "t" ) );
127
128 setPrintReport( cmd.hasOption( "r") );
129
130 return true;
131 }
132
133 public void setLimit( int limit )
134 {
135 this.limit = limit;
136 }
137
138 public void setThreadCount( long threadCount )
139 {
140 this.threadCount = threadCount;
141 }
142
143 protected SoapUIOptions initCommandLineOptions()
144 {
145 SoapUIOptions options = new SoapUIOptions( "loadtestrunner" );
146 options.addOption( "e", true, "Sets the endpoint" );
147 options.addOption( "s", true, "Sets the testsuite" );
148 options.addOption( "c", true, "Sets the testcase" );
149 options.addOption( "l", true, "Sets the loadtest" );
150 options.addOption( "u", true, "Sets the username" );
151 options.addOption( "p", true, "Sets the password" );
152 options.addOption( "w", true, "Sets the WSS password type, either 'Text' or 'Digest'" );
153 options.addOption( "d", true, "Sets the domain" );
154 options.addOption( "h", true, "Sets the host" );
155 options.addOption( "m", true, "Overrides the LoadTest Limit" );
156 options.addOption( "h", true, "Overrides the LoadTest ThreadCount" );
157 options.addOption( "r", false, "Exports statistics and testlogs for each LoadTest run" );
158 options.addOption( "f", true, "Sets the output folder to export to" );
159 options.addOption( "t", true, "Sets the soapui-settings.xml file to use" );
160 return options;
161 }
162
163 public SoapUILoadTestRunner()
164 {
165 this( TITLE );
166 }
167
168 public SoapUILoadTestRunner( String title )
169 {
170 super( title );
171 }
172
173 public void setLoadTest(String loadTest)
174 {
175 this.loadTest = loadTest;
176 }
177
178 public void setOutputFolder(String outputFolder)
179 {
180 this.outputFolder = outputFolder;
181 }
182
183 public void setPrintReport(boolean printReport)
184 {
185 this.printReport = printReport;
186 }
187
188 /***
189 * Sets the host to use by all test-requests, the existing endpoint port and path will be used
190 *
191 * @param host the host to use by all requests
192 */
193
194 public void setHost(String host)
195 {
196 this.host = host;
197 }
198
199 /***
200 * Sets the domain to use for any authentications
201 *
202 * @param domain the domain to use for any authentications
203 */
204
205 public void setDomain(String domain)
206 {
207 this.domain = domain;
208 }
209
210 /***
211 * Sets the password to use for any authentications
212 *
213 * @param domain the password to use for any authentications
214 */
215
216 public void setPassword(String password)
217 {
218 this.password = password;
219 }
220
221 /***
222 * Sets the WSS password-type to use for any authentications. Setting this will result
223 * in the addition of WS-Security UsernamePassword tokens to any outgoing request containing
224 * the specified username and password.
225 *
226 * @param wssPasswordType the wss-password type to use, either 'Text' or 'Digest'
227 */
228
229 public void setWssPasswordType( String wssPasswordType )
230 {
231 this.wssPasswordType = wssPasswordType;
232 }
233
234 /***
235 * Sets the username to use for any authentications
236 *
237 * @param domain the username to use for any authentications
238 */
239
240 public void setUsername(String username)
241 {
242 this.username = username;
243 }
244
245 /***
246 * Runs the testcases as configured with setXXX methods
247 *
248 * @throws Exception thrown if any tests fail
249 */
250
251 public boolean runRunner() throws Exception
252 {
253 if( SoapUI.getSettings().getBoolean( UISettings.DONT_DISABLE_GROOVY_LOG ))
254 {
255 initGroovyLog();
256 }
257
258 String projectFile = getProjectFile();
259
260 WsdlProject project = new WsdlProject( projectFile );
261 if( project.isDisabled() )
262 throw new Exception( "Failed to load soapUI project file [" + projectFile + "]" );
263 int suiteCount = 0;
264
265 for( int c = 0; c < project.getTestSuiteCount(); c++ )
266 {
267 if( testSuite == null ||
268 project.getTestSuiteAt( c ).getName().equalsIgnoreCase( testSuite ))
269 {
270 runSuite( project.getTestSuiteAt( c ));
271 suiteCount++;
272 }
273 }
274
275 if( suiteCount == 0 )
276 {
277 log.warn( "No test-suites matched argument [" + testSuite + "]" );
278 }
279 else if( testCaseCount == 0 )
280 {
281 log.warn( "No test-cases matched argument [" + testCase + "]" );
282 }
283 else if( loadTestCount == 0 )
284 {
285 log.warn( "No load-tests matched argument [" + loadTest + "]" );
286 }
287 else if( !failedTests.isEmpty() )
288 {
289 log.info( failedTests.size() + " load tests failed:" );
290 for( LoadTestRunner loadTestRunner : failedTests )
291 {
292 log.info( loadTestRunner.getLoadTest().getName() + ": " + loadTestRunner.getReason() );
293 }
294
295 throw new SoapUIException( "LoadTests failed" );
296 }
297
298 return true;
299 }
300
301 /***
302 * Run tests in the specified TestSuite
303 *
304 * @param suite the TestSuite to run
305 */
306
307 public void runSuite(TestSuite suite)
308 {
309 long start = System.currentTimeMillis();
310 for( int c = 0; c < suite.getTestCaseCount(); c++ )
311 {
312 String name = suite.getTestCaseAt( c ).getName();
313 if( testCase == null ||
314 name.equalsIgnoreCase( testCase ))
315 {
316 runTestCase( suite.getTestCaseAt( c ));
317 testCaseCount++;
318 }
319 else
320 log.info( "Skipping testcase [" + name + "], filter is [" + testCase + "]");
321 }
322 log.info( "soapUI suite [" + suite.getName() + "] finished in " + (System.currentTimeMillis()-start) + "ms" );
323 }
324
325 /***
326 * Runs the specified TestCase
327 *
328 * @param testCase the testcase to run
329 */
330
331 private void runTestCase(TestCase testCase)
332 {
333 for( int c = 0; c < testCase.getLoadTestCount(); c++ )
334 {
335 String name = testCase.getLoadTestAt( c ).getName();
336 if( loadTest == null || loadTest.equalsIgnoreCase( name ))
337 {
338 runWsdlLoadTest( (WsdlLoadTest) testCase.getLoadTestAt( c ));
339 loadTestCount++;
340 }
341 }
342 }
343
344 /***
345 * Runs the specified LoadTest
346 *
347 * @param loadTest the loadTest to run
348 */
349
350 private void runWsdlLoadTest(WsdlLoadTest loadTest)
351 {
352 try
353 {
354 log.info( "Running LoadTest [" + loadTest.getName() + "]" );
355 if( limit >= 0 )
356 {
357 log.info( "Overriding limit [" + loadTest.getTestLimit() + "] with specified [" + limit + "]" );
358 loadTest.setTestLimit( limit );
359 }
360
361 if( threadCount >= 0 )
362 {
363 log.info( "Overriding threadCount [" + loadTest.getThreadCount() + "] with specified [" + threadCount + "]" );
364 loadTest.setThreadCount( threadCount );
365 }
366
367 loadTest.addLoadTestRunListener(this);
368 LoadTestRunner runner = loadTest.run();
369
370
371 while (runner.getStatus() == LoadTestRunner.Status.RUNNING )
372 {
373 log.info( "LoadTest [" + loadTest.getName() + "] progress: " + runner.getProgress() + ", " +
374 runner.getRunningThreadCount() );
375 Thread.sleep(1000);
376 }
377
378 log.info( "LoadTest [" + loadTest.getName() + "] finished with status " + runner.getStatus().toString() );
379
380 if( printReport )
381 {
382 log.info( "Exporting log and statistics for LoadTest [" + loadTest.getName() + "]" );
383
384 loadTest.getStatisticsModel().finish();
385
386 exportLog( loadTest );
387 exportStatistics( loadTest );
388 }
389 }
390 catch (Exception e)
391 {
392 SoapUI.logError( e );
393 log.error( e );
394 }
395 }
396
397 private void exportStatistics(WsdlLoadTest loadTest) throws IOException
398 {
399 ExportStatisticsAction exportStatisticsAction = new ExportStatisticsAction( loadTest.getStatisticsModel() );
400 String statisticsFileName = loadTest.getName() + "-statistics.txt";
401 if( outputFolder != null )
402 {
403 ensureOutputFolder();
404 statisticsFileName = outputFolder + File.separator + statisticsFileName;
405 }
406
407 int cnt = exportStatisticsAction.exportToFile( new File( statisticsFileName ));
408 log.info( "Exported " + cnt + " statistics to [" + statisticsFileName + "]" );
409 }
410
411 private void exportLog(WsdlLoadTest loadTest) throws IOException
412 {
413
414 LoadTestLog loadTestLog = loadTest.getLoadTestLog();
415 ExportLoadTestLogAction exportLoadTestLogAction = new ExportLoadTestLogAction(loadTestLog, null);
416 String logFileName = loadTest.getName() + "-log.txt";
417 if( outputFolder != null )
418 {
419 ensureOutputFolder();
420 logFileName = outputFolder + File.separator + logFileName;
421 }
422
423 int cnt = exportLoadTestLogAction.exportToFile( new File( logFileName ));
424 log.info( "Exported " + cnt + " log items to [" + logFileName + "]" );
425
426 int errorCnt = 0;
427 for( int c = 0; c < loadTestLog.getSize(); c++ )
428 {
429 LoadTestLogEntry entry = (LoadTestLogEntry) loadTestLog.getElementAt( c );
430
431 if( entry != null && entry.isError() )
432 {
433 String entryFileName = loadTest.getName() + "-error-" + errorCnt++ + "-entry.txt";
434 if( outputFolder != null )
435 {
436 ensureOutputFolder();
437 entryFileName = outputFolder + File.separator + entryFileName;
438 }
439
440 try
441 {
442 entry.exportToFile( entryFileName );
443 }
444 catch (Exception e)
445 {
446 SoapUI.logError( e );
447 }
448 }
449 }
450 log.info( "Exported " + errorCnt + " error results" );
451 }
452
453 private void ensureOutputFolder()
454 {
455 File folder = new File( outputFolder );
456 if( !folder.exists())
457 folder.mkdirs();
458 }
459
460 /***
461 * Sets the testcase to run
462 *
463 * @param testCase the testcase to run
464 */
465
466 public void setTestCase(String testCase)
467 {
468 this.testCase = testCase;
469 }
470
471 /***
472 * Sets the endpoint to use for all test requests
473 *
474 * @param endpoint the endpoint to use for all test requests
475 */
476
477 public void setEndpoint(String endpoint)
478 {
479 this.endpoint = endpoint.trim();
480 }
481
482 /***
483 * Sets the TestSuite to run. If not set all TestSuites in the specified project file are run
484 *
485 * @param testSuite the testSuite to run.
486 */
487
488 public void setTestSuite(String testSuite)
489 {
490 this.testSuite = testSuite;
491 }
492
493 public void afterLoadTest(LoadTestRunner loadTestRunner, LoadTestRunContext context)
494 {
495 if( loadTestRunner.getStatus() == LoadTestRunner.Status.FAILED )
496 {
497 failedTests.add( loadTestRunner );
498 }
499 }
500
501 public void afterTestCase(LoadTestRunner loadTestRunner, LoadTestRunContext context, TestRunner testRunner, TestRunContext runContext)
502 {
503 }
504
505 public void afterTestStep(LoadTestRunner loadTestRunner, LoadTestRunContext context, TestRunner testRunner, TestRunContext runContext, TestStepResult testStepResult)
506 {
507 }
508
509 public void beforeLoadTest(LoadTestRunner loadTestRunner, LoadTestRunContext context)
510 {
511 }
512
513 public void beforeTestCase(LoadTestRunner loadTestRunner, LoadTestRunContext context, TestRunner testRunner, TestRunContext runContext)
514 {
515 }
516
517 public void beforeTestStep(LoadTestRunner loadTestRunner, LoadTestRunContext context, TestRunner testRunner, TestRunContext runContext, TestStep testStep)
518 {
519 if( testStep instanceof WsdlTestRequestStep )
520 {
521 WsdlTestRequestStep requestStep = (WsdlTestRequestStep) testStep;
522 if( endpoint != null && endpoint.length() > 0 )
523 {
524 requestStep.getTestRequest().setEndpoint( endpoint );
525 }
526
527 if( host != null && host.length() > 0 )
528 {
529 try
530 {
531 String ep = Tools.replaceHost( requestStep.getTestRequest().getEndpoint(), host );
532 requestStep.getTestRequest().setEndpoint( ep );
533 }
534 catch (Exception e)
535 {
536 log.error( "Failed to set host on endpoint", e );
537 }
538 }
539
540 if( username != null && username.length() > 0 )
541 {
542 requestStep.getTestRequest().setUsername( username );
543 }
544
545 if( password != null && password.length() > 0 )
546 {
547 requestStep.getTestRequest().setPassword( password );
548 }
549
550 if( domain != null && domain.length() > 0 )
551 {
552 requestStep.getTestRequest().setDomain( domain );
553 }
554
555 if( wssPasswordType != null && wssPasswordType.length() > 0 )
556 {
557 requestStep.getTestRequest().setWssPasswordType( wssPasswordType.equals( "Digest" ) ?
558 WsdlTestRequest.PW_TYPE_DIGEST : WsdlTestRequest.PW_TYPE_TEXT );
559 }
560 }
561 }
562
563 public void loadTestStarted(LoadTestRunner loadTestRunner, LoadTestRunContext context)
564 {
565 }
566
567 public void loadTestStopped(LoadTestRunner loadTestRunner, LoadTestRunContext context)
568 {
569 }
570 }