View Javadoc

1   /*
2    *  soapUI, copyright (C) 2004-2008 eviware.com 
3    *
4    *  soapUI is free software; you can redistribute it and/or modify it under the 
5    *  terms of version 2.1 of the GNU Lesser General Public License as published by 
6    *  the Free Software Foundation.
7    *
8    *  soapUI is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without 
9    *  even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 
10   *  See the GNU Lesser General Public License for more details at gnu.org.
11   */
12  
13  package com.eviware.soapui.tools;
14  
15  import com.eviware.soapui.SoapUI;
16  import com.eviware.soapui.impl.wsdl.WsdlProject;
17  import com.eviware.soapui.impl.wsdl.WsdlTestSuite;
18  import com.eviware.soapui.impl.wsdl.testcase.WsdlTestCase;
19  import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestRequestStepResult;
20  import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestStep;
21  import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestStepResult;
22  import com.eviware.soapui.model.iface.Attachment;
23  import com.eviware.soapui.model.project.ProjectFactoryRegistry;
24  import com.eviware.soapui.model.propertyexpansion.DefaultPropertyExpansionContext;
25  import com.eviware.soapui.model.propertyexpansion.PropertyExpansionContext;
26  import com.eviware.soapui.model.testsuite.*;
27  import com.eviware.soapui.model.testsuite.Assertable.AssertionStatus;
28  import com.eviware.soapui.model.testsuite.AssertionError;
29  import com.eviware.soapui.model.testsuite.TestRunner.Status;
30  import com.eviware.soapui.model.testsuite.TestStepResult.TestStepStatus;
31  import com.eviware.soapui.model.testsuite.TestSuite.TestSuiteRunType;
32  import com.eviware.soapui.report.JUnitReportCollector;
33  import com.eviware.soapui.support.StringUtils;
34  import com.eviware.soapui.support.Tools;
35  import com.eviware.soapui.support.types.StringToObjectMap;
36  import org.apache.commons.cli.CommandLine;
37  
38  import java.io.File;
39  import java.io.FileOutputStream;
40  import java.io.PrintWriter;
41  import java.io.StringWriter;
42  import java.util.*;
43  
44  /***
45   * Standalone test-runner used from maven-plugin, can also be used from
46   * command-line (see xdocs) or directly from other classes.
47   * <p>
48   * For standalone usage, set the project file (with setProjectFile) and other
49   * desired properties before calling run
50   * </p>
51   *
52   * @author Ole.Matzura
53   */
54  
55  public class SoapUITestCaseRunner extends AbstractSoapUITestRunner
56  {
57     public static final String TITLE = "soapUI " + SoapUI.SOAPUI_VERSION + " TestCase Runner";
58  
59     private String testSuite;
60     private String testCase;
61     private List<TestAssertion> assertions = new ArrayList<TestAssertion>();
62     private Map<TestAssertion, WsdlTestStepResult> assertionResults = new HashMap<TestAssertion, WsdlTestStepResult>();
63     private List<TestCase> runningTests = new ArrayList<TestCase>();
64     private List<TestCase> failedTests = new ArrayList<TestCase>();
65  
66     private int testSuiteCount;
67     private int testCaseCount;
68     private int testStepCount;
69     private int testAssertionCount;
70  
71     private boolean printReport;
72     private boolean exportAll;
73     private boolean ignoreErrors;
74     private boolean junitReport;
75     private int exportCount;
76     private JUnitReportCollector reportCollector;
77     private String wssPasswordType;
78     private WsdlProject project;
79  
80     private String projectPassword;
81  
82     /***
83      * Runs the tests in the specified soapUI project file, see soapUI xdocs for
84      * details.
85      *
86      * @param args
87      * @throws Exception
88      */
89  
90     public static void main( String[] args ) throws Exception
91     {
92        new SoapUITestCaseRunner().runFromCommandLine( args );
93     }
94  
95     protected boolean processCommandLine( CommandLine cmd )
96     {
97        if( cmd.hasOption( "e" ) )
98           setEndpoint( cmd.getOptionValue( "e" ) );
99  
100       if( cmd.hasOption( "s" ) )
101          setTestSuite( getCommandLineOptionSubstSpace( cmd, "s" ) );
102 
103       if( cmd.hasOption( "c" ) )
104          setTestCase( getCommandLineOptionSubstSpace( cmd, "c" ) );
105 
106       if( cmd.hasOption( "u" ) )
107          setUsername( cmd.getOptionValue( "u" ) );
108 
109       if( cmd.hasOption( "p" ) )
110          setPassword( cmd.getOptionValue( "p" ) );
111 
112       if( cmd.hasOption( "w" ) )
113          setWssPasswordType( cmd.getOptionValue( "w" ) );
114 
115       if( cmd.hasOption( "d" ) )
116          setDomain( cmd.getOptionValue( "d" ) );
117 
118       if( cmd.hasOption( "h" ) )
119          setHost( cmd.getOptionValue( "h" ) );
120 
121       if( cmd.hasOption( "f" ) )
122          setOutputFolder( getCommandLineOptionSubstSpace( cmd, "f" ) );
123 
124       if( cmd.hasOption( "t" ) )
125          setSettingsFile( getCommandLineOptionSubstSpace( cmd, "t" ) );
126 
127       if( cmd.hasOption( "x" ) )
128       {
129          setProjectPassword( cmd.getOptionValue( "x" ) );
130       }
131 
132       if( cmd.hasOption( "v" ) )
133       {
134          setSoapUISettingsPassword( cmd.getOptionValue( "v" ) );
135       }
136 
137       if( cmd.hasOption( "D" ) )
138       {
139          setSystemProperties( cmd.getOptionValues( "D" ) );
140       }
141 
142       if( cmd.hasOption( "G" ) )
143       {
144          setGlobalProperties( cmd.getOptionValues( "G" ) );
145       }
146 
147       setEnableUI( cmd.hasOption( "i" ) );
148       setPrintReport( cmd.hasOption( "r" ) );
149       setExportAll( cmd.hasOption( "a" ) );
150       setJUnitReport( cmd.hasOption( "j" ) );
151 
152       return true;
153    }
154 
155    public void setProjectPassword( String projectPassword )
156    {
157       this.projectPassword = projectPassword;
158    }
159 
160    public String getProjectPassword()
161    {
162       return projectPassword;
163    }
164 
165    protected SoapUIOptions initCommandLineOptions()
166    {
167       SoapUIOptions options = new SoapUIOptions( "testrunner" );
168       options.addOption( "e", true, "Sets the endpoint" );
169       options.addOption( "s", true, "Sets the testsuite" );
170       options.addOption( "c", true, "Sets the testcase" );
171       options.addOption( "u", true, "Sets the username" );
172       options.addOption( "p", true, "Sets the password" );
173       options.addOption( "w", true, "Sets the WSS password type, either 'Text' or 'Digest'" );
174       options.addOption( "i", false, "Enables Swing UI for scripts" );
175       options.addOption( "d", true, "Sets the domain" );
176       options.addOption( "h", true, "Sets the host" );
177       options.addOption( "r", false, "Prints a small summary report" );
178       options.addOption( "f", true, "Sets the output folder to export results to" );
179       options.addOption( "j", false, "Sets the output to include JUnit XML reports" );
180       options.addOption( "a", false, "Turns on exporting of all results" );
181       options.addOption( "t", true, "Sets the soapui-settings.xml file to use" );
182       options.addOption( "x", true, "Sets project password for decryption if project is encrypted" );
183       options.addOption( "v", true, "Sets password for soapui-settings.xml file" );
184       options.addOption( "D", true, "Sets system property with name=value");
185       options.addOption( "G", true, "Sets global property with name=value" );
186       
187       return options;
188    }
189 
190    /***
191     * Add console appender to groovy log
192     */
193 
194    public void setExportAll( boolean exportAll )
195    {
196       this.exportAll = exportAll;
197    }
198 
199    public void setJUnitReport( boolean junitReport )
200    {
201       this.junitReport = junitReport;
202       if( junitReport )
203          reportCollector = new JUnitReportCollector();
204    }
205 
206    public SoapUITestCaseRunner()
207    {
208       super( SoapUITestCaseRunner.TITLE );
209    }
210 
211    public SoapUITestCaseRunner( String title )
212    {
213       super( title );
214    }
215 
216    /***
217     * Controls if a short test summary should be printed after the test runs
218     *
219     * @param printReport a flag controlling if a summary should be printed
220     */
221 
222    public void setPrintReport( boolean printReport )
223    {
224       this.printReport = printReport;
225    }
226 
227    public void setIgnoreError( boolean ignoreErrors )
228    {
229       this.ignoreErrors = ignoreErrors;
230    }
231 
232    public boolean runRunner() throws Exception
233    {
234       initGroovyLog();
235 
236       assertions.clear();
237 
238       String projectFile = getProjectFile();
239 
240 //		project = new WsdlProject(projectFile, getProjectPassword());
241       project = (WsdlProject) ProjectFactoryRegistry.getProjectFactory( "wsdl" ).createNew( projectFile, getProjectPassword() );
242 
243       if( project.isDisabled() )
244          throw new Exception( "Failed to load soapUI project file [" + projectFile + "]" );
245 
246       initProject();
247       ensureOutputFolder( project );
248 
249       log.info( "Running soapUI tests in project [" + project.getName() + "]" );
250 
251       long startTime = System.nanoTime();
252 
253       // start by listening to all testcases.. (since one testcase can call
254       // another)
255       for( int c = 0; c < project.getTestSuiteCount(); c++ )
256       {
257          TestSuite suite = project.getTestSuiteAt( c );
258          for( int i = 0; i < suite.getTestCaseCount(); i++ )
259          {
260             TestCase tc = suite.getTestCaseAt( i );
261             addListeners( tc );
262          }
263       }
264 
265       // now run tests..
266       for( int c = 0; c < project.getTestSuiteCount(); c++ )
267       {
268          WsdlTestSuite ts = project.getTestSuiteAt( c );
269          if( !ts.isDisabled() && ( testSuite == null || ts.getName().equalsIgnoreCase( testSuite ) ) )
270          {
271             runSuite( ts );
272             testSuiteCount++;
273 
274             // wait for tests to finish if running in parallell mode
275             if( !runningTests.isEmpty() )
276                log.info( "Waiting for " + runningTests.size() + " tests to finish" );
277 
278             while( !runningTests.isEmpty() )
279             {
280                Thread.sleep( 100 );
281             }
282          }
283       }
284 
285       long timeTaken = ( System.nanoTime() - startTime ) / 1000000;
286 
287       if( printReport )
288       {
289          printReport( timeTaken );
290       }
291 
292       exportReports( project );
293 
294       if( assertions.size() > 0 || failedTests.size() > 0 && !ignoreErrors )
295       {
296          throwFailureException();
297       }
298 
299       return true;
300    }
301 
302    protected void initProject() throws Exception
303    {
304    }
305 
306    protected void exportReports( WsdlProject project ) throws Exception
307    {
308       if( junitReport )
309       {
310          exportJUnitReports( reportCollector, getAbsoluteOutputFolder( project ) );
311       }
312    }
313 
314    protected void addListeners( TestCase tc )
315    {
316       tc.addTestRunListener( this );
317       if( junitReport )
318          tc.addTestRunListener( reportCollector );
319    }
320 
321    protected void throwFailureException() throws Exception
322    {
323       StringBuffer buf = new StringBuffer();
324 
325       for( int c = 0; c < assertions.size(); c++ )
326       {
327          TestAssertion assertion = assertions.get( c );
328          Assertable assertable = assertion.getAssertable();
329          if( assertable instanceof WsdlTestStep )
330             failedTests.remove( ( (WsdlTestStep) assertable ).getTestCase() );
331 
332          buf.append( assertion.getName() + " in [" + assertable.getModelItem().getName() + "] failed;\n" );
333          buf.append( Arrays.toString( assertion.getErrors() ) + "\n" );
334 
335          WsdlTestStepResult result = assertionResults.get( assertion );
336          StringWriter stringWriter = new StringWriter();
337          PrintWriter writer = new PrintWriter( stringWriter );
338          result.writeTo( writer );
339          buf.append( stringWriter.toString() );
340       }
341 
342       while( !failedTests.isEmpty() )
343       {
344          buf.append( "TestCase [" + failedTests.remove( 0 ).getName() + "] failed without assertions\n" );
345       }
346 
347       throw new Exception( buf.toString() );
348    }
349 
350    public void exportJUnitReports( JUnitReportCollector collector, String folder ) throws Exception
351    {
352       collector.saveReports( folder == null ? "" : folder );
353    }
354 
355    public void printReport( long timeTaken )
356    {
357       System.out.println();
358       System.out.println( "SoapUI " + SoapUI.SOAPUI_VERSION + " TestCaseRunner Summary" );
359       System.out.println( "-----------------------------" );
360       System.out.println( "Time Taken: " + timeTaken + "ms" );
361       System.out.println( "Total TestSuites: " + testSuiteCount );
362       System.out.println( "Total TestCases: " + testCaseCount + " (" + failedTests.size() + " failed)" );
363       System.out.println( "Total TestSteps: " + testStepCount );
364       System.out.println( "Total Request Assertions: " + testAssertionCount );
365       System.out.println( "Total Failed Assertions: " + assertions.size() );
366       System.out.println( "Total Exported Results: " + exportCount );
367    }
368 
369    /***
370     * Run tests in the specified TestSuite
371     *
372     * @param suite the TestSuite to run
373     */
374 
375    public void runSuite( WsdlTestSuite suite )
376    {
377       log.info( ( "Running soapUI suite [" + suite.getName() + "], runType = " + suite.getRunType() ) );
378       PropertyExpansionContext context = new DefaultPropertyExpansionContext( suite );
379       long start = System.currentTimeMillis();
380 
381       try
382       {
383          suite.runSetupScript( context );
384          for( int c = 0; c < suite.getTestCaseCount(); c++ )
385          {
386             WsdlTestCase tc = suite.getTestCaseAt( c );
387 
388             String name = tc.getName();
389             if( testCase == null || name.equalsIgnoreCase( testCase ) )
390             {
391                if( !tc.isDisabled() )
392                {
393                   runTestCase( tc );
394                }
395                else
396                {
397                   log.info( "Skipping disabled testcase [" + name + "]" );
398                }
399             }
400             else
401             {
402                log.info( "Skipping testcase [" + name + "], filter is [" + testCase + "]" );
403             }
404          }
405       }
406       catch( Exception e )
407       {
408          e.printStackTrace();
409       }
410       finally
411       {
412          try
413          {
414             suite.runTearDownScript( context );
415          }
416          catch( Exception e )
417          {
418             e.printStackTrace();
419          }
420       }
421 
422       log.info( "soapUI suite [" + suite.getName() + "] finished in " + ( System.currentTimeMillis() - start ) + "ms" );
423    }
424 
425    /***
426     * Runs the specified TestCase
427     *
428     * @param testCase the testcase to run
429     */
430 
431    private void runTestCase( TestCase testCase )
432    {
433       runningTests.add( testCase );
434       testCase.run( new StringToObjectMap(), testCase.getTestSuite().getRunType() == TestSuiteRunType.PARALLEL );
435    }
436 
437    /***
438     * Sets the testcase to run
439     *
440     * @param testCase the testcase to run
441     */
442 
443    public void setTestCase( String testCase )
444    {
445       this.testCase = testCase;
446    }
447 
448    /***
449     * Sets the TestSuite to run. If not set all TestSuites in the specified
450     * project file are run
451     *
452     * @param testSuite the testSuite to run.
453     */
454 
455    public void setTestSuite( String testSuite )
456    {
457       this.testSuite = testSuite;
458    }
459 
460    public void beforeRun( TestRunner testRunner, TestRunContext runContext )
461    {
462       log.info( "Running soapUI testcase [" + testRunner.getTestCase().getName() + "]" );
463    }
464 
465    public void beforeStep( TestRunner testRunner, TestRunContext runContext )
466    {
467       super.beforeStep( testRunner, runContext );
468 
469       TestStep currentStep = runContext.getCurrentStep();
470       log.info( "running step [" + currentStep.getName() + "]" );
471    }
472 
473    public void afterStep( TestRunner testRunner, TestRunContext runContext, TestStepResult result )
474    {
475       super.afterStep( testRunner, runContext, result );
476       TestStep currentStep = runContext.getCurrentStep();
477 
478       if( currentStep instanceof Assertable )
479       {
480          Assertable requestStep = (Assertable) currentStep;
481          for( int c = 0; c < requestStep.getAssertionCount(); c++ )
482          {
483             TestAssertion assertion = requestStep.getAssertionAt( c );
484             log.info( "Assertion [" + assertion.getName() + "] has status " + assertion.getStatus() );
485             if( assertion.getStatus() == AssertionStatus.FAILED )
486             {
487                for( AssertionError error : assertion.getErrors() )
488                   log.error( "ASSERTION FAILED -> " + error.getMessage() );
489 
490                assertions.add( assertion );
491                assertionResults.put( assertion, (WsdlTestStepResult) result );
492             }
493 
494             testAssertionCount++;
495          }
496       }
497 
498       String countPropertyName = currentStep.getName() + " run count";
499       Long count = (Long) runContext.getProperty( countPropertyName );
500       if( count == null )
501       {
502          count = new Long( 0 );
503       }
504 
505       runContext.setProperty( countPropertyName, new Long( count.longValue() + 1 ) );
506 
507       if( result.getStatus() == TestStepStatus.FAILED || exportAll )
508       {
509          try
510          {
511             TestCase tc = currentStep.getTestCase();
512             String nameBase = StringUtils.createFileName( tc.getTestSuite().getName(), '_' ) + "-"
513                     + StringUtils.createFileName( tc.getName(), '_' ) + "-"
514                     + StringUtils.createFileName( currentStep.getName(), '_' ) + "-" + count.longValue() + "-"
515                     + result.getStatus();
516 
517             String absoluteOutputFolder = getAbsoluteOutputFolder( project );
518             String fileName = absoluteOutputFolder + File.separator + nameBase + ".txt";
519 
520             if( result.getStatus() == TestStepStatus.FAILED )
521                log.error( currentStep.getName() + " failed, exporting to [" + fileName + "]" );
522 
523             PrintWriter writer = new PrintWriter( fileName );
524             result.writeTo( writer );
525             writer.close();
526 
527             // write attachments
528             if( result instanceof WsdlTestRequestStepResult )
529             {
530                Attachment[] attachments = ( (WsdlTestRequestStepResult) result ).getResponseAttachments();
531                if( attachments != null && attachments.length > 0 )
532                {
533                   for( int c = 0; c < attachments.length; c++ )
534                   {
535                      fileName = nameBase + "-attachment-" + ( c + 1 ) + ".";
536 
537                      Attachment attachment = attachments[c];
538                      String contentType = attachment.getContentType();
539                      if( !"application/octet-stream".equals( contentType ) && contentType != null
540                              && contentType.indexOf( '/' ) != -1 )
541                      {
542                         fileName += contentType.substring( contentType.lastIndexOf( '/' ) + 1 );
543                      }
544                      else
545                      {
546                         fileName += "dat";
547                      }
548 
549                      fileName = absoluteOutputFolder + File.separator + fileName;
550 
551                      FileOutputStream outFile = new FileOutputStream( fileName );
552                      Tools.writeAll( outFile, attachment.getInputStream() );
553                      outFile.close();
554                   }
555                }
556             }
557 
558             exportCount++;
559          }
560          catch( Exception e )
561          {
562             log.error( "Error saving failed result: " + e, e );
563          }
564       }
565 
566       testStepCount++;
567    }
568 
569    public void afterRun( TestRunner testRunner, TestRunContext runContext )
570    {
571       log.info( "Finished running soapUI testcase [" + testRunner.getTestCase().getName() + "], time taken: "
572               + testRunner.getTimeTaken() + "ms, status: " + testRunner.getStatus() );
573 
574       if( testRunner.getStatus() == Status.FAILED )
575       {
576          failedTests.add( testRunner.getTestCase() );
577       }
578 
579       runningTests.remove( testRunner.getTestCase() );
580 
581       testCaseCount++;
582    }
583 
584    protected WsdlProject getProject()
585    {
586       return project;
587    }
588 }