1
2
3
4
5
6
7
8
9
10
11
12
13 package com.eviware.soapui.tools;
14
15 import java.io.File;
16 import java.io.IOException;
17 import java.util.ArrayList;
18 import java.util.List;
19
20 import org.apache.commons.cli.CommandLine;
21
22 import com.eviware.soapui.SoapUI;
23 import com.eviware.soapui.impl.wsdl.WsdlProject;
24 import com.eviware.soapui.impl.wsdl.loadtest.WsdlLoadTest;
25 import com.eviware.soapui.impl.wsdl.loadtest.data.actions.ExportLoadTestLogAction;
26 import com.eviware.soapui.impl.wsdl.loadtest.data.actions.ExportStatisticsAction;
27 import com.eviware.soapui.impl.wsdl.loadtest.log.LoadTestLog;
28 import com.eviware.soapui.impl.wsdl.loadtest.log.LoadTestLogEntry;
29 import com.eviware.soapui.model.project.ProjectFactoryRegistry;
30 import com.eviware.soapui.model.testsuite.LoadTestRunContext;
31 import com.eviware.soapui.model.testsuite.LoadTestRunListener;
32 import com.eviware.soapui.model.testsuite.LoadTestRunner;
33 import com.eviware.soapui.model.testsuite.TestCase;
34 import com.eviware.soapui.model.testsuite.TestCaseRunContext;
35 import com.eviware.soapui.model.testsuite.TestCaseRunner;
36 import com.eviware.soapui.model.testsuite.TestStep;
37 import com.eviware.soapui.model.testsuite.TestStepResult;
38 import com.eviware.soapui.model.testsuite.TestSuite;
39 import com.eviware.soapui.model.testsuite.TestRunner.Status;
40 import com.eviware.soapui.settings.UISettings;
41 import com.eviware.soapui.support.SoapUIException;
42 import com.eviware.soapui.support.StringUtils;
43
44 /***
45 * Standalone test-runner used from maven-plugin, can also be used from
46 * command-line (see xdocs) or directly from other classes.
47 * <p>
48 * For standalone usage, set the project file (with setProjectFile) and other
49 * desired properties before calling run
50 * </p>
51 *
52 * @author Ole.Matzura
53 */
54
55 public class SoapUILoadTestRunner extends AbstractSoapUITestRunner implements LoadTestRunListener
56 {
57 private String testSuite;
58 private String testCase;
59 private String loadTest;
60 private boolean printReport;
61 private List<LoadTestRunner> failedTests = new ArrayList<LoadTestRunner>();
62 private int testCaseCount;
63 private int loadTestCount;
64 private int limit = -1;
65 private long threadCount = -1;
66 private boolean saveAfterRun;
67
68 public static String TITLE = "soapUI " + SoapUI.SOAPUI_VERSION + " LoadTest Runner";
69
70 /***
71 * Runs the loadtests in the specified soapUI project file, see soapUI xdocs
72 * for details.
73 *
74 * @param args
75 * @throws Exception
76 */
77
78 public static void main( String[] args )
79 {
80 System.exit( new SoapUILoadTestRunner().runFromCommandLine( args ));
81 }
82
83 protected boolean processCommandLine( CommandLine cmd )
84 {
85 if( cmd.hasOption( "e" ) )
86 setEndpoint( cmd.getOptionValue( "e" ) );
87
88 if( cmd.hasOption( "s" ) )
89 setTestSuite( getCommandLineOptionSubstSpace( cmd, "s" ) );
90
91 if( cmd.hasOption( "c" ) )
92 setTestCase( cmd.getOptionValue( "c" ) );
93
94 if( cmd.hasOption( "l" ) )
95 setLoadTest( cmd.getOptionValue( "l" ) );
96
97 if( cmd.hasOption( "u" ) )
98 setUsername( cmd.getOptionValue( "u" ) );
99
100 if( cmd.hasOption( "p" ) )
101 setPassword( cmd.getOptionValue( "p" ) );
102
103 if( cmd.hasOption( "w" ) )
104 setWssPasswordType( cmd.getOptionValue( "w" ) );
105
106 if( cmd.hasOption( "d" ) )
107 setDomain( cmd.getOptionValue( "d" ) );
108
109 if( cmd.hasOption( "h" ) )
110 setHost( cmd.getOptionValue( "h" ) );
111
112 if( cmd.hasOption( "m" ) )
113 setLimit( Integer.parseInt( cmd.getOptionValue( "m" ) ) );
114
115 if( cmd.hasOption( "n" ) )
116 setThreadCount( Integer.parseInt( cmd.getOptionValue( "n" ) ) );
117
118 if( cmd.hasOption( "f" ) )
119 setOutputFolder( getCommandLineOptionSubstSpace( cmd, "f" ) );
120
121 if( cmd.hasOption( "t" ) )
122 setSettingsFile( getCommandLineOptionSubstSpace( cmd, "t" ) );
123
124 setPrintReport( cmd.hasOption( "r" ) );
125 setSaveAfterRun( cmd.hasOption( "S" ) );
126
127 if( cmd.hasOption( "x" ) )
128 {
129 setProjectPassword( cmd.getOptionValue( "x" ) );
130 }
131
132 if( cmd.hasOption( "v" ) )
133 {
134 setSoapUISettingsPassword( cmd.getOptionValue( "v" ) );
135 }
136
137 if( cmd.hasOption( "D" ) )
138 {
139 setSystemProperties( cmd.getOptionValues( "D" ) );
140 }
141
142 if( cmd.hasOption( "G" ) )
143 {
144 setGlobalProperties( cmd.getOptionValues( "G" ) );
145 }
146
147 if( cmd.hasOption( "P" ) )
148 {
149 setProjectProperties( cmd.getOptionValues( "P" ) );
150 }
151
152 return true;
153 }
154
155 public void setLimit( int limit )
156 {
157 this.limit = limit;
158 }
159
160 public void setThreadCount( long threadCount )
161 {
162 this.threadCount = threadCount;
163 }
164
165 protected SoapUIOptions initCommandLineOptions()
166 {
167 SoapUIOptions options = new SoapUIOptions( "loadtestrunner" );
168 options.addOption( "e", true, "Sets the endpoint" );
169 options.addOption( "s", true, "Sets the testsuite" );
170 options.addOption( "c", true, "Sets the testcase" );
171 options.addOption( "l", true, "Sets the loadtest" );
172 options.addOption( "u", true, "Sets the username" );
173 options.addOption( "p", true, "Sets the password" );
174 options.addOption( "w", true, "Sets the WSS password type, either 'Text' or 'Digest'" );
175 options.addOption( "d", true, "Sets the domain" );
176 options.addOption( "h", true, "Sets the host" );
177 options.addOption( "m", true, "Overrides the LoadTest Limit" );
178 options.addOption( "n", true, "Overrides the LoadTest ThreadCount" );
179 options.addOption( "r", false, "Exports statistics and testlogs for each LoadTest run" );
180 options.addOption( "f", true, "Sets the output folder to export to" );
181 options.addOption( "t", true, "Sets the soapui-settings.xml file to use" );
182 options.addOption( "x", true, "Sets project password for decryption if project is encrypted" );
183 options.addOption( "v", true, "Sets password for soapui-settings.xml file" );
184 options.addOption( "D", true, "Sets system property with name=value" );
185 options.addOption( "G", true, "Sets global property with name=value" );
186 options.addOption( "P", true, "Sets or overrides project property with name=value" );
187 options.addOption( "S", false, "Saves the project after running the tests" );
188
189 return options;
190 }
191
192 public SoapUILoadTestRunner()
193 {
194 this( TITLE );
195 }
196
197 public SoapUILoadTestRunner( String title )
198 {
199 super( title );
200 }
201
202 public void setLoadTest( String loadTest )
203 {
204 this.loadTest = loadTest;
205 }
206
207 public void setPrintReport( boolean printReport )
208 {
209 this.printReport = printReport;
210 }
211
212 public void setSaveAfterRun( boolean saveAfterRun )
213 {
214 this.saveAfterRun = saveAfterRun;
215 }
216
217 /***
218 * Runs the testcases as configured with setXXX methods
219 *
220 * @throws Exception
221 * thrown if any tests fail
222 */
223
224 public boolean runRunner() throws Exception
225 {
226 if( SoapUI.getSettings().getBoolean( UISettings.DONT_DISABLE_GROOVY_LOG ) )
227 {
228 initGroovyLog();
229 }
230
231 String projectFile = getProjectFile();
232
233
234
235 WsdlProject project = ( WsdlProject )ProjectFactoryRegistry.getProjectFactory( "wsdl" ).createNew( projectFile,
236 getProjectPassword() );
237
238 if( project.isDisabled() )
239 throw new Exception( "Failed to load soapUI project file [" + projectFile + "]" );
240
241 initProjectProperties( project );
242
243 int suiteCount = 0;
244
245 for( int c = 0; c < project.getTestSuiteCount(); c++ )
246 {
247 if( testSuite == null || project.getTestSuiteAt( c ).getName().equalsIgnoreCase( testSuite ) )
248 {
249 runSuite( project.getTestSuiteAt( c ) );
250 suiteCount++ ;
251 }
252 }
253
254 if( suiteCount == 0 )
255 {
256 log.warn( "No test-suites matched argument [" + testSuite + "]" );
257 }
258 else if( testCaseCount == 0 )
259 {
260 log.warn( "No test-cases matched argument [" + testCase + "]" );
261 }
262 else if( loadTestCount == 0 )
263 {
264 log.warn( "No load-tests matched argument [" + loadTest + "]" );
265 }
266 else
267 {
268 if( saveAfterRun && !project.isRemote() )
269 {
270 try
271 {
272 project.save();
273 }
274 catch( Throwable t )
275 {
276 log.error( "Failed to save project", t );
277 }
278 }
279
280 if( !failedTests.isEmpty() )
281 {
282 log.info( failedTests.size() + " load tests failed:" );
283 for( LoadTestRunner loadTestRunner : failedTests )
284 {
285 log.info( loadTestRunner.getLoadTest().getName() + ": " + loadTestRunner.getReason() );
286 }
287
288 throw new SoapUIException( "LoadTests failed" );
289 }
290 }
291
292 return true;
293 }
294
295 /***
296 * Run tests in the specified TestSuite
297 *
298 * @param suite
299 * the TestSuite to run
300 */
301
302 public void runSuite( TestSuite suite )
303 {
304 long start = System.currentTimeMillis();
305 for( int c = 0; c < suite.getTestCaseCount(); c++ )
306 {
307 String name = suite.getTestCaseAt( c ).getName();
308 if( testCase == null || name.equalsIgnoreCase( testCase ) )
309 {
310 runTestCase( suite.getTestCaseAt( c ) );
311 testCaseCount++ ;
312 }
313 else
314 log.info( "Skipping testcase [" + name + "], filter is [" + testCase + "]" );
315 }
316 log.info( "soapUI suite [" + suite.getName() + "] finished in " + ( System.currentTimeMillis() - start ) + "ms" );
317 }
318
319 /***
320 * Runs the specified TestCase
321 *
322 * @param testCase
323 * the testcase to run
324 */
325
326 private void runTestCase( TestCase testCase )
327 {
328 for( int c = 0; c < testCase.getLoadTestCount(); c++ )
329 {
330 String name = testCase.getLoadTestAt( c ).getName();
331 if( loadTest == null || loadTest.equalsIgnoreCase( name ) )
332 {
333 runWsdlLoadTest( ( WsdlLoadTest )testCase.getLoadTestAt( c ) );
334 loadTestCount++ ;
335 }
336 }
337 }
338
339 /***
340 * Runs the specified LoadTest
341 *
342 * @param loadTest
343 * the loadTest to run
344 */
345
346 protected void runWsdlLoadTest( WsdlLoadTest loadTest )
347 {
348 try
349 {
350 log.info( "Running LoadTest [" + loadTest.getName() + "]" );
351 if( limit >= 0 )
352 {
353 log.info( "Overriding limit [" + loadTest.getTestLimit() + "] with specified [" + limit + "]" );
354 loadTest.setTestLimit( limit );
355 }
356
357 if( threadCount >= 0 )
358 {
359 log.info( "Overriding threadCount [" + loadTest.getThreadCount() + "] with specified [" + threadCount
360 + "]" );
361 loadTest.setThreadCount( threadCount );
362 }
363
364 loadTest.addLoadTestRunListener( this );
365 LoadTestRunner runner = loadTest.run();
366
367
368 while( !runner.hasStopped() )
369 {
370 if( runner.getStatus() == Status.RUNNING )
371 {
372 log.info( "LoadTest [" + loadTest.getName() + "] progress: " + runner.getProgress() + ", "
373 + runner.getRunningThreadCount() );
374 }
375 Thread.sleep( 1000 );
376 }
377
378 log.info( "LoadTest [" + loadTest.getName() + "] finished with status " + runner.getStatus().toString() );
379
380 if( printReport )
381 {
382 log.info( "Exporting log and statistics for LoadTest [" + loadTest.getName() + "]" );
383
384 loadTest.getStatisticsModel().finish();
385
386 exportLog( loadTest );
387 exportStatistics( loadTest );
388 }
389 }
390 catch( Exception e )
391 {
392 SoapUI.logError( e );
393 log.error( e );
394 }
395 }
396
397 private void exportStatistics( WsdlLoadTest loadTest ) throws IOException
398 {
399 ExportStatisticsAction exportStatisticsAction = new ExportStatisticsAction( loadTest.getStatisticsModel() );
400 String statisticsFileName = StringUtils.createFileName( loadTest.getName(), '_' ) + "-statistics.txt";
401 if( getOutputFolder() != null )
402 {
403 ensureOutputFolder( loadTest );
404 statisticsFileName = getAbsoluteOutputFolder( loadTest ) + File.separator + statisticsFileName;
405 }
406
407 int cnt = exportStatisticsAction.exportToFile( new File( statisticsFileName ) );
408 log.info( "Exported " + cnt + " statistics to [" + statisticsFileName + "]" );
409 }
410
411 private void exportLog( WsdlLoadTest loadTest ) throws IOException
412 {
413
414 LoadTestLog loadTestLog = loadTest.getLoadTestLog();
415 ExportLoadTestLogAction exportLoadTestLogAction = new ExportLoadTestLogAction( loadTestLog, null );
416 String logFileName = StringUtils.createFileName( loadTest.getName(), '_' ) + "-log.txt";
417 if( getOutputFolder() != null )
418 {
419 ensureOutputFolder( loadTest );
420 logFileName = getAbsoluteOutputFolder( loadTest ) + File.separator + logFileName;
421 }
422
423 int cnt = exportLoadTestLogAction.exportToFile( new File( logFileName ) );
424 log.info( "Exported " + cnt + " log items to [" + logFileName + "]" );
425
426 int errorCnt = 0;
427 for( int c = 0; c < loadTestLog.getSize(); c++ )
428 {
429 LoadTestLogEntry entry = ( LoadTestLogEntry )loadTestLog.getElementAt( c );
430
431 if( entry != null && entry.isError() )
432 {
433 String entryFileName = StringUtils.createFileName( loadTest.getName(), '_' ) + "-error-" + errorCnt++
434 + "-entry.txt";
435 if( getOutputFolder() != null )
436 {
437 ensureOutputFolder( loadTest );
438 entryFileName = getAbsoluteOutputFolder( loadTest ) + File.separator + entryFileName;
439 }
440
441 try
442 {
443 entry.exportToFile( entryFileName );
444 }
445 catch( Exception e )
446 {
447 SoapUI.logError( e );
448 }
449 }
450 }
451 log.info( "Exported " + errorCnt + " error results" );
452 }
453
454 /***
455 * Sets the testcase to run
456 *
457 * @param testCase
458 * the testcase to run
459 */
460
461 public void setTestCase( String testCase )
462 {
463 this.testCase = testCase;
464 }
465
466 /***
467 * Sets the TestSuite to run. If not set all TestSuites in the specified
468 * project file are run
469 *
470 * @param testSuite
471 * the testSuite to run.
472 */
473
474 public void setTestSuite( String testSuite )
475 {
476 this.testSuite = testSuite;
477 }
478
479 public void afterLoadTest( LoadTestRunner loadTestRunner, LoadTestRunContext context )
480 {
481 if( loadTestRunner.getStatus() == LoadTestRunner.Status.FAILED )
482 {
483 failedTests.add( loadTestRunner );
484 }
485 }
486
487 public void afterTestCase( LoadTestRunner loadTestRunner, LoadTestRunContext context, TestCaseRunner testRunner,
488 TestCaseRunContext runContext )
489 {
490 }
491
492 public void afterTestStep( LoadTestRunner loadTestRunner, LoadTestRunContext context, TestCaseRunner testRunner,
493 TestCaseRunContext runContext, TestStepResult testStepResult )
494 {
495 super.afterStep( testRunner, runContext, testStepResult );
496 }
497
498 public void beforeLoadTest( LoadTestRunner loadTestRunner, LoadTestRunContext context )
499 {
500 }
501
502 public void beforeTestCase( LoadTestRunner loadTestRunner, LoadTestRunContext context, TestCaseRunner testRunner,
503 TestCaseRunContext runContext )
504 {
505 }
506
507 public void beforeTestStep( LoadTestRunner loadTestRunner, LoadTestRunContext context, TestCaseRunner testRunner,
508 TestCaseRunContext runContext, TestStep testStep )
509 {
510 super.beforeStep( testRunner, runContext, testStep );
511 }
512
513 public void loadTestStarted( LoadTestRunner loadTestRunner, LoadTestRunContext context )
514 {
515 }
516
517 public void loadTestStopped( LoadTestRunner loadTestRunner, LoadTestRunContext context )
518 {
519 }
520 }