View Javadoc

1   /*
2    *  soapUI, copyright (C) 2004-2008 eviware.com 
3    *
4    *  soapUI is free software; you can redistribute it and/or modify it under the 
5    *  terms of version 2.1 of the GNU Lesser General Public License as published by 
6    *  the Free Software Foundation.
7    *
8    *  soapUI is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without 
9    *  even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 
10   *  See the GNU Lesser General Public License for more details at gnu.org.
11   */
12  
13  package com.eviware.soapui.tools;
14  
15  import com.eviware.soapui.SoapUI;
16  import com.eviware.soapui.impl.wsdl.WsdlProject;
17  import com.eviware.soapui.impl.wsdl.loadtest.WsdlLoadTest;
18  import com.eviware.soapui.impl.wsdl.loadtest.data.actions.ExportLoadTestLogAction;
19  import com.eviware.soapui.impl.wsdl.loadtest.data.actions.ExportStatisticsAction;
20  import com.eviware.soapui.impl.wsdl.loadtest.log.LoadTestLog;
21  import com.eviware.soapui.impl.wsdl.loadtest.log.LoadTestLogEntry;
22  import com.eviware.soapui.model.project.ProjectFactoryRegistry;
23  import com.eviware.soapui.model.testsuite.*;
24  import com.eviware.soapui.settings.UISettings;
25  import com.eviware.soapui.support.SoapUIException;
26  import com.eviware.soapui.support.StringUtils;
27  import org.apache.commons.cli.CommandLine;
28  
29  import java.io.File;
30  import java.io.IOException;
31  import java.util.ArrayList;
32  import java.util.List;
33  
34  /***
35   * Standalone test-runner used from maven-plugin, can also be used from command-line (see xdocs) or
36   * directly from other classes.
37   * <p>
38   * For standalone usage, set the project file (with setProjectFile) and other desired properties before
39   * calling run</p>
40   *
41   * @author Ole.Matzura
42   */
43  
44  public class SoapUILoadTestRunner extends AbstractSoapUITestRunner implements LoadTestRunListener
45  {
46     private String testSuite;
47     private String testCase;
48     private String loadTest;
49     private boolean printReport;
50     private List<LoadTestRunner> failedTests = new ArrayList<LoadTestRunner>();
51     private int testCaseCount;
52     private int loadTestCount;
53     private int limit = -1;
54     private long threadCount = -1;
55  
56     public static String TITLE = "soapUI " + SoapUI.SOAPUI_VERSION + " LoadTest Runner";
57  
58     /***
59      * Runs the loadtests in the specified soapUI project file, see soapUI xdocs for details.
60      *
61      * @param args
62      * @throws Exception
63      */
64  
65     public static void main( String[] args )
66     {
67        new SoapUILoadTestRunner().runFromCommandLine( args );
68     }
69  
70     protected boolean processCommandLine( CommandLine cmd )
71     {
72        if( cmd.hasOption( "e" ) )
73           setEndpoint( cmd.getOptionValue( "e" ) );
74  
75        if( cmd.hasOption( "s" ) )
76           setTestSuite( getCommandLineOptionSubstSpace( cmd, "s" ) );
77  
78        if( cmd.hasOption( "c" ) )
79           setTestCase( cmd.getOptionValue( "c" ) );
80  
81        if( cmd.hasOption( "l" ) )
82           setLoadTest( cmd.getOptionValue( "l" ) );
83  
84        if( cmd.hasOption( "u" ) )
85           setUsername( cmd.getOptionValue( "u" ) );
86  
87        if( cmd.hasOption( "p" ) )
88           setPassword( cmd.getOptionValue( "p" ) );
89  
90        if( cmd.hasOption( "w" ) )
91           setWssPasswordType( cmd.getOptionValue( "w" ) );
92  
93        if( cmd.hasOption( "d" ) )
94           setDomain( cmd.getOptionValue( "d" ) );
95  
96        if( cmd.hasOption( "h" ) )
97           setHost( cmd.getOptionValue( "h" ) );
98  
99        if( cmd.hasOption( "m" ) )
100          setLimit( Integer.parseInt( cmd.getOptionValue( "m" ) ) );
101 
102       if( cmd.hasOption( "n" ) )
103          setThreadCount( Integer.parseInt( cmd.getOptionValue( "n" ) ) );
104 
105       if( cmd.hasOption( "f" ) )
106          setOutputFolder( getCommandLineOptionSubstSpace( cmd, "f" ) );
107 
108       if( cmd.hasOption( "t" ) )
109          setSettingsFile( getCommandLineOptionSubstSpace( cmd, "t" ) );
110 
111       setPrintReport( cmd.hasOption( "r" ) );
112 
113       if( cmd.hasOption( "x" ) )
114       {
115          setProjectPassword( cmd.getOptionValue( "x" ) );
116       }
117 
118       if( cmd.hasOption( "v" ) )
119       {
120          setSoapUISettingsPassword( cmd.getOptionValue( "v" ) );
121       }
122 
123       if( cmd.hasOption( "D" ) )
124       {
125          setSystemProperties( cmd.getOptionValues( "D" ) );
126       }
127 
128       if( cmd.hasOption( "G" ) )
129       {
130          setGlobalProperties( cmd.getOptionValues( "G" ) );
131       }
132 
133       return true;
134    }
135 
136    public void setLimit( int limit )
137    {
138       this.limit = limit;
139    }
140 
141    public void setThreadCount( long threadCount )
142    {
143       this.threadCount = threadCount;
144    }
145 
146    protected SoapUIOptions initCommandLineOptions()
147    {
148       SoapUIOptions options = new SoapUIOptions( "loadtestrunner" );
149       options.addOption( "e", true, "Sets the endpoint" );
150       options.addOption( "s", true, "Sets the testsuite" );
151       options.addOption( "c", true, "Sets the testcase" );
152       options.addOption( "l", true, "Sets the loadtest" );
153       options.addOption( "u", true, "Sets the username" );
154       options.addOption( "p", true, "Sets the password" );
155       options.addOption( "w", true, "Sets the WSS password type, either 'Text' or 'Digest'" );
156       options.addOption( "d", true, "Sets the domain" );
157       options.addOption( "h", true, "Sets the host" );
158       options.addOption( "m", true, "Overrides the LoadTest Limit" );
159       options.addOption( "n", true, "Overrides the LoadTest ThreadCount" );
160       options.addOption( "r", false, "Exports statistics and testlogs for each LoadTest run" );
161       options.addOption( "f", true, "Sets the output folder to export to" );
162       options.addOption( "t", true, "Sets the soapui-settings.xml file to use" );
163       options.addOption( "x", true, "Sets project password for decryption if project is encrypted" );
164       options.addOption( "v", true, "Sets password for soapui-settings.xml file" );
165       options.addOption( "D", true, "Sets system property with name=value" );
166       options.addOption( "G", true, "Sets global property with name=value" );
167 
168       return options;
169    }
170 
171    public SoapUILoadTestRunner()
172    {
173       this( TITLE );
174    }
175 
176    public SoapUILoadTestRunner( String title )
177    {
178       super( title );
179    }
180 
181    public void setLoadTest( String loadTest )
182    {
183       this.loadTest = loadTest;
184    }
185 
186    public void setPrintReport( boolean printReport )
187    {
188       this.printReport = printReport;
189    }
190 
191    /***
192     * Runs the testcases as configured with setXXX methods
193     *
194     * @throws Exception thrown if any tests fail
195     */
196 
197    public boolean runRunner() throws Exception
198    {
199       if( SoapUI.getSettings().getBoolean( UISettings.DONT_DISABLE_GROOVY_LOG ) )
200       {
201          initGroovyLog();
202       }
203 
204       String projectFile = getProjectFile();
205 
206 //		WsdlProject project = new WsdlProject( projectFile, getProjectPassword() );
207       WsdlProject project = (WsdlProject) ProjectFactoryRegistry.getProjectFactory( "wsdl" ).createNew( projectFile, getProjectPassword() );
208 
209       if( project.isDisabled() )
210          throw new Exception( "Failed to load soapUI project file [" + projectFile + "]" );
211       int suiteCount = 0;
212 
213       for( int c = 0; c < project.getTestSuiteCount(); c++ )
214       {
215          if( testSuite == null ||
216                  project.getTestSuiteAt( c ).getName().equalsIgnoreCase( testSuite ) )
217          {
218             runSuite( project.getTestSuiteAt( c ) );
219             suiteCount++;
220          }
221       }
222 
223       if( suiteCount == 0 )
224       {
225          log.warn( "No test-suites matched argument [" + testSuite + "]" );
226       }
227       else if( testCaseCount == 0 )
228       {
229          log.warn( "No test-cases matched argument [" + testCase + "]" );
230       }
231       else if( loadTestCount == 0 )
232       {
233          log.warn( "No load-tests matched argument [" + loadTest + "]" );
234       }
235       else if( !failedTests.isEmpty() )
236       {
237          log.info( failedTests.size() + " load tests failed:" );
238          for( LoadTestRunner loadTestRunner : failedTests )
239          {
240             log.info( loadTestRunner.getLoadTest().getName() + ": " + loadTestRunner.getReason() );
241          }
242 
243          throw new SoapUIException( "LoadTests failed" );
244       }
245 
246       return true;
247    }
248 
249    /***
250     * Run tests in the specified TestSuite
251     *
252     * @param suite the TestSuite to run
253     */
254 
255    public void runSuite( TestSuite suite )
256    {
257       long start = System.currentTimeMillis();
258       for( int c = 0; c < suite.getTestCaseCount(); c++ )
259       {
260          String name = suite.getTestCaseAt( c ).getName();
261          if( testCase == null ||
262                  name.equalsIgnoreCase( testCase ) )
263          {
264             runTestCase( suite.getTestCaseAt( c ) );
265             testCaseCount++;
266          }
267          else
268             log.info( "Skipping testcase [" + name + "], filter is [" + testCase + "]" );
269       }
270       log.info( "soapUI suite [" + suite.getName() + "] finished in " + ( System.currentTimeMillis() - start ) + "ms" );
271    }
272 
273    /***
274     * Runs the specified TestCase
275     *
276     * @param testCase the testcase to run
277     */
278 
279    private void runTestCase( TestCase testCase )
280    {
281       for( int c = 0; c < testCase.getLoadTestCount(); c++ )
282       {
283          String name = testCase.getLoadTestAt( c ).getName();
284          if( loadTest == null || loadTest.equalsIgnoreCase( name ) )
285          {
286             runWsdlLoadTest( (WsdlLoadTest) testCase.getLoadTestAt( c ) );
287             loadTestCount++;
288          }
289       }
290    }
291 
292    /***
293     * Runs the specified LoadTest
294     *
295     * @param loadTest the loadTest to run
296     */
297 
298    private void runWsdlLoadTest( WsdlLoadTest loadTest )
299    {
300       try
301       {
302          log.info( "Running LoadTest [" + loadTest.getName() + "]" );
303          if( limit >= 0 )
304          {
305             log.info( "Overriding limit [" + loadTest.getTestLimit() + "] with specified [" + limit + "]" );
306             loadTest.setTestLimit( limit );
307          }
308 
309          if( threadCount >= 0 )
310          {
311             log.info( "Overriding threadCount [" + loadTest.getThreadCount() + "] with specified [" + threadCount + "]" );
312             loadTest.setThreadCount( threadCount );
313          }
314 
315          loadTest.addLoadTestRunListener( this );
316          LoadTestRunner runner = loadTest.run();
317 
318          // wait for test to finish
319          while( runner.getStatus() == LoadTestRunner.Status.RUNNING )
320          {
321             log.info( "LoadTest [" + loadTest.getName() + "] progress: " + runner.getProgress() + ", " +
322                     runner.getRunningThreadCount() );
323             Thread.sleep( 1000 );
324          }
325 
326          log.info( "LoadTest [" + loadTest.getName() + "] finished with status " + runner.getStatus().toString() );
327 
328          if( printReport )
329          {
330             log.info( "Exporting log and statistics for LoadTest [" + loadTest.getName() + "]" );
331 
332             loadTest.getStatisticsModel().finish();
333 
334             exportLog( loadTest );
335             exportStatistics( loadTest );
336          }
337       }
338       catch( Exception e )
339       {
340          SoapUI.logError( e );
341          log.error( e );
342       }
343    }
344 
345    private void exportStatistics( WsdlLoadTest loadTest ) throws IOException
346    {
347       ExportStatisticsAction exportStatisticsAction = new ExportStatisticsAction( loadTest.getStatisticsModel() );
348       String statisticsFileName = StringUtils.createFileName( loadTest.getName(), '_' ) + "-statistics.txt";
349       if( getOutputFolder() != null )
350       {
351          ensureOutputFolder( loadTest );
352          statisticsFileName = getAbsoluteOutputFolder( loadTest ) + File.separator + statisticsFileName;
353       }
354 
355       int cnt = exportStatisticsAction.exportToFile( new File( statisticsFileName ) );
356       log.info( "Exported " + cnt + " statistics to [" + statisticsFileName + "]" );
357    }
358 
359    private void exportLog( WsdlLoadTest loadTest ) throws IOException
360    {
361       // export log first
362       LoadTestLog loadTestLog = loadTest.getLoadTestLog();
363       ExportLoadTestLogAction exportLoadTestLogAction = new ExportLoadTestLogAction( loadTestLog, null );
364       String logFileName = StringUtils.createFileName( loadTest.getName(), '_' ) + "-log.txt";
365       if( getOutputFolder() != null )
366       {
367          ensureOutputFolder( loadTest );
368          logFileName = getAbsoluteOutputFolder( loadTest ) + File.separator + logFileName;
369       }
370 
371       int cnt = exportLoadTestLogAction.exportToFile( new File( logFileName ) );
372       log.info( "Exported " + cnt + " log items to [" + logFileName + "]" );
373 
374       int errorCnt = 0;
375       for( int c = 0; c < loadTestLog.getSize(); c++ )
376       {
377          LoadTestLogEntry entry = (LoadTestLogEntry) loadTestLog.getElementAt( c );
378 
379          if( entry != null && entry.isError() )
380          {
381             String entryFileName = StringUtils.createFileName( loadTest.getName(), '_' ) + "-error-" + errorCnt++ + "-entry.txt";
382             if( getOutputFolder() != null )
383             {
384                ensureOutputFolder( loadTest );
385                entryFileName = getAbsoluteOutputFolder( loadTest ) + File.separator + entryFileName;
386             }
387 
388             try
389             {
390                entry.exportToFile( entryFileName );
391             }
392             catch( Exception e )
393             {
394                SoapUI.logError( e );
395             }
396          }
397       }
398       log.info( "Exported " + errorCnt + " error results" );
399    }
400 
401    /***
402     * Sets the testcase to run
403     *
404     * @param testCase the testcase to run
405     */
406 
407    public void setTestCase( String testCase )
408    {
409       this.testCase = testCase;
410    }
411 
412    /***
413     * Sets the TestSuite to run. If not set all TestSuites in the specified project file are run
414     *
415     * @param testSuite the testSuite to run.
416     */
417 
418    public void setTestSuite( String testSuite )
419    {
420       this.testSuite = testSuite;
421    }
422 
423    public void afterLoadTest( LoadTestRunner loadTestRunner, LoadTestRunContext context )
424    {
425       if( loadTestRunner.getStatus() == LoadTestRunner.Status.FAILED )
426       {
427          failedTests.add( loadTestRunner );
428       }
429    }
430 
431    public void afterTestCase( LoadTestRunner loadTestRunner, LoadTestRunContext context, TestRunner testRunner, TestRunContext runContext )
432    {
433    }
434 
435    public void afterTestStep( LoadTestRunner loadTestRunner, LoadTestRunContext context, TestRunner testRunner, TestRunContext runContext, TestStepResult testStepResult )
436    {
437       super.afterStep( testRunner, runContext, testStepResult );
438    }
439 
440    public void beforeLoadTest( LoadTestRunner loadTestRunner, LoadTestRunContext context )
441    {
442    }
443 
444    public void beforeTestCase( LoadTestRunner loadTestRunner, LoadTestRunContext context, TestRunner testRunner, TestRunContext runContext )
445    {
446    }
447 
448    public void beforeTestStep( LoadTestRunner loadTestRunner, LoadTestRunContext context, TestRunner testRunner, TestRunContext runContext, TestStep testStep )
449    {
450       super.beforeStep( testRunner, runContext );
451    }
452 
453    public void loadTestStarted( LoadTestRunner loadTestRunner, LoadTestRunContext context )
454    {
455    }
456 
457    public void loadTestStopped( LoadTestRunner loadTestRunner, LoadTestRunContext context )
458    {
459    }
460 }