1
2
3
4
5
6
7
8
9
10
11
12
13 package com.eviware.soapui.tools;
14
15 import java.io.File;
16 import java.io.IOException;
17 import java.util.ArrayList;
18 import java.util.List;
19
20 import org.apache.commons.cli.CommandLine;
21
22 import com.eviware.soapui.SoapUI;
23 import com.eviware.soapui.impl.wsdl.WsdlProject;
24 import com.eviware.soapui.impl.wsdl.loadtest.WsdlLoadTest;
25 import com.eviware.soapui.impl.wsdl.loadtest.data.actions.ExportLoadTestLogAction;
26 import com.eviware.soapui.impl.wsdl.loadtest.data.actions.ExportStatisticsAction;
27 import com.eviware.soapui.impl.wsdl.loadtest.log.LoadTestLog;
28 import com.eviware.soapui.impl.wsdl.loadtest.log.LoadTestLogEntry;
29 import com.eviware.soapui.model.project.ProjectFactoryRegistry;
30 import com.eviware.soapui.model.testsuite.LoadTestRunContext;
31 import com.eviware.soapui.model.testsuite.LoadTestRunListener;
32 import com.eviware.soapui.model.testsuite.LoadTestRunner;
33 import com.eviware.soapui.model.testsuite.TestCase;
34 import com.eviware.soapui.model.testsuite.TestCaseRunContext;
35 import com.eviware.soapui.model.testsuite.TestCaseRunner;
36 import com.eviware.soapui.model.testsuite.TestStep;
37 import com.eviware.soapui.model.testsuite.TestStepResult;
38 import com.eviware.soapui.model.testsuite.TestSuite;
39 import com.eviware.soapui.settings.UISettings;
40 import com.eviware.soapui.support.SoapUIException;
41 import com.eviware.soapui.support.StringUtils;
42
43 /***
44 * Standalone test-runner used from maven-plugin, can also be used from
45 * command-line (see xdocs) or directly from other classes.
46 * <p>
47 * For standalone usage, set the project file (with setProjectFile) and other
48 * desired properties before calling run
49 * </p>
50 *
51 * @author Ole.Matzura
52 */
53
54 public class SoapUILoadTestRunner extends AbstractSoapUITestRunner implements LoadTestRunListener
55 {
56 private String testSuite;
57 private String testCase;
58 private String loadTest;
59 private boolean printReport;
60 private List<LoadTestRunner> failedTests = new ArrayList<LoadTestRunner>();
61 private int testCaseCount;
62 private int loadTestCount;
63 private int limit = -1;
64 private long threadCount = -1;
65 private boolean saveAfterRun;
66
67 public static String TITLE = "soapUI " + SoapUI.SOAPUI_VERSION + " LoadTest Runner";
68
69 /***
70 * Runs the loadtests in the specified soapUI project file, see soapUI xdocs
71 * for details.
72 *
73 * @param args
74 * @throws Exception
75 */
76
77 public static void main( String[] args )
78 {
79 System.exit( new SoapUILoadTestRunner().runFromCommandLine( args ));
80 }
81
82 protected boolean processCommandLine( CommandLine cmd )
83 {
84 if( cmd.hasOption( "e" ) )
85 setEndpoint( cmd.getOptionValue( "e" ) );
86
87 if( cmd.hasOption( "s" ) )
88 setTestSuite( getCommandLineOptionSubstSpace( cmd, "s" ) );
89
90 if( cmd.hasOption( "c" ) )
91 setTestCase( cmd.getOptionValue( "c" ) );
92
93 if( cmd.hasOption( "l" ) )
94 setLoadTest( cmd.getOptionValue( "l" ) );
95
96 if( cmd.hasOption( "u" ) )
97 setUsername( cmd.getOptionValue( "u" ) );
98
99 if( cmd.hasOption( "p" ) )
100 setPassword( cmd.getOptionValue( "p" ) );
101
102 if( cmd.hasOption( "w" ) )
103 setWssPasswordType( cmd.getOptionValue( "w" ) );
104
105 if( cmd.hasOption( "d" ) )
106 setDomain( cmd.getOptionValue( "d" ) );
107
108 if( cmd.hasOption( "h" ) )
109 setHost( cmd.getOptionValue( "h" ) );
110
111 if( cmd.hasOption( "m" ) )
112 setLimit( Integer.parseInt( cmd.getOptionValue( "m" ) ) );
113
114 if( cmd.hasOption( "n" ) )
115 setThreadCount( Integer.parseInt( cmd.getOptionValue( "n" ) ) );
116
117 if( cmd.hasOption( "f" ) )
118 setOutputFolder( getCommandLineOptionSubstSpace( cmd, "f" ) );
119
120 if( cmd.hasOption( "t" ) )
121 setSettingsFile( getCommandLineOptionSubstSpace( cmd, "t" ) );
122
123 setPrintReport( cmd.hasOption( "r" ) );
124 setSaveAfterRun( cmd.hasOption( "S" ) );
125
126 if( cmd.hasOption( "x" ) )
127 {
128 setProjectPassword( cmd.getOptionValue( "x" ) );
129 }
130
131 if( cmd.hasOption( "v" ) )
132 {
133 setSoapUISettingsPassword( cmd.getOptionValue( "v" ) );
134 }
135
136 if( cmd.hasOption( "D" ) )
137 {
138 setSystemProperties( cmd.getOptionValues( "D" ) );
139 }
140
141 if( cmd.hasOption( "G" ) )
142 {
143 setGlobalProperties( cmd.getOptionValues( "G" ) );
144 }
145
146 if( cmd.hasOption( "P" ) )
147 {
148 setProjectProperties( cmd.getOptionValues( "P" ) );
149 }
150
151 return true;
152 }
153
154 public void setLimit( int limit )
155 {
156 this.limit = limit;
157 }
158
159 public void setThreadCount( long threadCount )
160 {
161 this.threadCount = threadCount;
162 }
163
164 protected SoapUIOptions initCommandLineOptions()
165 {
166 SoapUIOptions options = new SoapUIOptions( "loadtestrunner" );
167 options.addOption( "e", true, "Sets the endpoint" );
168 options.addOption( "s", true, "Sets the testsuite" );
169 options.addOption( "c", true, "Sets the testcase" );
170 options.addOption( "l", true, "Sets the loadtest" );
171 options.addOption( "u", true, "Sets the username" );
172 options.addOption( "p", true, "Sets the password" );
173 options.addOption( "w", true, "Sets the WSS password type, either 'Text' or 'Digest'" );
174 options.addOption( "d", true, "Sets the domain" );
175 options.addOption( "h", true, "Sets the host" );
176 options.addOption( "m", true, "Overrides the LoadTest Limit" );
177 options.addOption( "n", true, "Overrides the LoadTest ThreadCount" );
178 options.addOption( "r", false, "Exports statistics and testlogs for each LoadTest run" );
179 options.addOption( "f", true, "Sets the output folder to export to" );
180 options.addOption( "t", true, "Sets the soapui-settings.xml file to use" );
181 options.addOption( "x", true, "Sets project password for decryption if project is encrypted" );
182 options.addOption( "v", true, "Sets password for soapui-settings.xml file" );
183 options.addOption( "D", true, "Sets system property with name=value" );
184 options.addOption( "G", true, "Sets global property with name=value" );
185 options.addOption( "P", true, "Sets or overrides project property with name=value" );
186 options.addOption( "S", false, "Saves the project after running the tests" );
187
188 return options;
189 }
190
191 public SoapUILoadTestRunner()
192 {
193 this( TITLE );
194 }
195
196 public SoapUILoadTestRunner( String title )
197 {
198 super( title );
199 }
200
201 public void setLoadTest( String loadTest )
202 {
203 this.loadTest = loadTest;
204 }
205
206 public void setPrintReport( boolean printReport )
207 {
208 this.printReport = printReport;
209 }
210
211 public void setSaveAfterRun( boolean saveAfterRun )
212 {
213 this.saveAfterRun = saveAfterRun;
214 }
215
216 /***
217 * Runs the testcases as configured with setXXX methods
218 *
219 * @throws Exception
220 * thrown if any tests fail
221 */
222
223 public boolean runRunner() throws Exception
224 {
225 if( SoapUI.getSettings().getBoolean( UISettings.DONT_DISABLE_GROOVY_LOG ) )
226 {
227 initGroovyLog();
228 }
229
230 String projectFile = getProjectFile();
231
232
233
234 WsdlProject project = ( WsdlProject )ProjectFactoryRegistry.getProjectFactory( "wsdl" ).createNew( projectFile,
235 getProjectPassword() );
236
237 if( project.isDisabled() )
238 throw new Exception( "Failed to load soapUI project file [" + projectFile + "]" );
239
240 initProjectProperties( project );
241
242 int suiteCount = 0;
243
244 for( int c = 0; c < project.getTestSuiteCount(); c++ )
245 {
246 if( testSuite == null || project.getTestSuiteAt( c ).getName().equalsIgnoreCase( testSuite ) )
247 {
248 runSuite( project.getTestSuiteAt( c ) );
249 suiteCount++ ;
250 }
251 }
252
253 if( suiteCount == 0 )
254 {
255 log.warn( "No test-suites matched argument [" + testSuite + "]" );
256 }
257 else if( testCaseCount == 0 )
258 {
259 log.warn( "No test-cases matched argument [" + testCase + "]" );
260 }
261 else if( loadTestCount == 0 )
262 {
263 log.warn( "No load-tests matched argument [" + loadTest + "]" );
264 }
265 else
266 {
267 if( saveAfterRun && !project.isRemote() )
268 {
269 try
270 {
271 project.save();
272 }
273 catch( Throwable t )
274 {
275 log.error( "Failed to save project", t );
276 }
277 }
278
279 if( !failedTests.isEmpty() )
280 {
281 log.info( failedTests.size() + " load tests failed:" );
282 for( LoadTestRunner loadTestRunner : failedTests )
283 {
284 log.info( loadTestRunner.getLoadTest().getName() + ": " + loadTestRunner.getReason() );
285 }
286
287 throw new SoapUIException( "LoadTests failed" );
288 }
289 }
290
291 return true;
292 }
293
294 /***
295 * Run tests in the specified TestSuite
296 *
297 * @param suite
298 * the TestSuite to run
299 */
300
301 public void runSuite( TestSuite suite )
302 {
303 long start = System.currentTimeMillis();
304 for( int c = 0; c < suite.getTestCaseCount(); c++ )
305 {
306 String name = suite.getTestCaseAt( c ).getName();
307 if( testCase == null || name.equalsIgnoreCase( testCase ) )
308 {
309 runTestCase( suite.getTestCaseAt( c ) );
310 testCaseCount++ ;
311 }
312 else
313 log.info( "Skipping testcase [" + name + "], filter is [" + testCase + "]" );
314 }
315 log.info( "soapUI suite [" + suite.getName() + "] finished in " + ( System.currentTimeMillis() - start ) + "ms" );
316 }
317
318 /***
319 * Runs the specified TestCase
320 *
321 * @param testCase
322 * the testcase to run
323 */
324
325 private void runTestCase( TestCase testCase )
326 {
327 for( int c = 0; c < testCase.getLoadTestCount(); c++ )
328 {
329 String name = testCase.getLoadTestAt( c ).getName();
330 if( loadTest == null || loadTest.equalsIgnoreCase( name ) )
331 {
332 runWsdlLoadTest( ( WsdlLoadTest )testCase.getLoadTestAt( c ) );
333 loadTestCount++ ;
334 }
335 }
336 }
337
338 /***
339 * Runs the specified LoadTest
340 *
341 * @param loadTest
342 * the loadTest to run
343 */
344
345 protected void runWsdlLoadTest( WsdlLoadTest loadTest )
346 {
347 try
348 {
349 log.info( "Running LoadTest [" + loadTest.getName() + "]" );
350 if( limit >= 0 )
351 {
352 log.info( "Overriding limit [" + loadTest.getTestLimit() + "] with specified [" + limit + "]" );
353 loadTest.setTestLimit( limit );
354 }
355
356 if( threadCount >= 0 )
357 {
358 log
359 .info( "Overriding threadCount [" + loadTest.getThreadCount() + "] with specified [" + threadCount
360 + "]" );
361 loadTest.setThreadCount( threadCount );
362 }
363
364 loadTest.addLoadTestRunListener( this );
365 LoadTestRunner runner = loadTest.run();
366
367
368 while( runner.getStatus() == LoadTestRunner.Status.RUNNING )
369 {
370 log.info( "LoadTest [" + loadTest.getName() + "] progress: " + runner.getProgress() + ", "
371 + runner.getRunningThreadCount() );
372 Thread.sleep( 1000 );
373 }
374
375 log.info( "LoadTest [" + loadTest.getName() + "] finished with status " + runner.getStatus().toString() );
376
377 if( printReport )
378 {
379 log.info( "Exporting log and statistics for LoadTest [" + loadTest.getName() + "]" );
380
381 loadTest.getStatisticsModel().finish();
382
383 exportLog( loadTest );
384 exportStatistics( loadTest );
385 }
386 }
387 catch( Exception e )
388 {
389 SoapUI.logError( e );
390 log.error( e );
391 }
392 }
393
394 private void exportStatistics( WsdlLoadTest loadTest ) throws IOException
395 {
396 ExportStatisticsAction exportStatisticsAction = new ExportStatisticsAction( loadTest.getStatisticsModel() );
397 String statisticsFileName = StringUtils.createFileName( loadTest.getName(), '_' ) + "-statistics.txt";
398 if( getOutputFolder() != null )
399 {
400 ensureOutputFolder( loadTest );
401 statisticsFileName = getAbsoluteOutputFolder( loadTest ) + File.separator + statisticsFileName;
402 }
403
404 int cnt = exportStatisticsAction.exportToFile( new File( statisticsFileName ) );
405 log.info( "Exported " + cnt + " statistics to [" + statisticsFileName + "]" );
406 }
407
408 private void exportLog( WsdlLoadTest loadTest ) throws IOException
409 {
410
411 LoadTestLog loadTestLog = loadTest.getLoadTestLog();
412 ExportLoadTestLogAction exportLoadTestLogAction = new ExportLoadTestLogAction( loadTestLog, null );
413 String logFileName = StringUtils.createFileName( loadTest.getName(), '_' ) + "-log.txt";
414 if( getOutputFolder() != null )
415 {
416 ensureOutputFolder( loadTest );
417 logFileName = getAbsoluteOutputFolder( loadTest ) + File.separator + logFileName;
418 }
419
420 int cnt = exportLoadTestLogAction.exportToFile( new File( logFileName ) );
421 log.info( "Exported " + cnt + " log items to [" + logFileName + "]" );
422
423 int errorCnt = 0;
424 for( int c = 0; c < loadTestLog.getSize(); c++ )
425 {
426 LoadTestLogEntry entry = ( LoadTestLogEntry )loadTestLog.getElementAt( c );
427
428 if( entry != null && entry.isError() )
429 {
430 String entryFileName = StringUtils.createFileName( loadTest.getName(), '_' ) + "-error-" + errorCnt++
431 + "-entry.txt";
432 if( getOutputFolder() != null )
433 {
434 ensureOutputFolder( loadTest );
435 entryFileName = getAbsoluteOutputFolder( loadTest ) + File.separator + entryFileName;
436 }
437
438 try
439 {
440 entry.exportToFile( entryFileName );
441 }
442 catch( Exception e )
443 {
444 SoapUI.logError( e );
445 }
446 }
447 }
448 log.info( "Exported " + errorCnt + " error results" );
449 }
450
451 /***
452 * Sets the testcase to run
453 *
454 * @param testCase
455 * the testcase to run
456 */
457
458 public void setTestCase( String testCase )
459 {
460 this.testCase = testCase;
461 }
462
463 /***
464 * Sets the TestSuite to run. If not set all TestSuites in the specified
465 * project file are run
466 *
467 * @param testSuite
468 * the testSuite to run.
469 */
470
471 public void setTestSuite( String testSuite )
472 {
473 this.testSuite = testSuite;
474 }
475
476 public void afterLoadTest( LoadTestRunner loadTestRunner, LoadTestRunContext context )
477 {
478 if( loadTestRunner.getStatus() == LoadTestRunner.Status.FAILED )
479 {
480 failedTests.add( loadTestRunner );
481 }
482 }
483
484 public void afterTestCase( LoadTestRunner loadTestRunner, LoadTestRunContext context, TestCaseRunner testRunner,
485 TestCaseRunContext runContext )
486 {
487 }
488
489 public void afterTestStep( LoadTestRunner loadTestRunner, LoadTestRunContext context, TestCaseRunner testRunner,
490 TestCaseRunContext runContext, TestStepResult testStepResult )
491 {
492 super.afterStep( testRunner, runContext, testStepResult );
493 }
494
495 public void beforeLoadTest( LoadTestRunner loadTestRunner, LoadTestRunContext context )
496 {
497 }
498
499 public void beforeTestCase( LoadTestRunner loadTestRunner, LoadTestRunContext context, TestCaseRunner testRunner,
500 TestCaseRunContext runContext )
501 {
502 }
503
504 public void beforeTestStep( LoadTestRunner loadTestRunner, LoadTestRunContext context, TestCaseRunner testRunner,
505 TestCaseRunContext runContext, TestStep testStep )
506 {
507 super.beforeStep( testRunner, runContext, testStep );
508 }
509
510 public void loadTestStarted( LoadTestRunner loadTestRunner, LoadTestRunContext context )
511 {
512 }
513
514 public void loadTestStopped( LoadTestRunner loadTestRunner, LoadTestRunContext context )
515 {
516 }
517 }