View Javadoc

1   /*
2    *  soapUI, copyright (C) 2004-2008 eviware.com 
3    *
4    *  soapUI is free software; you can redistribute it and/or modify it under the 
5    *  terms of version 2.1 of the GNU Lesser General Public License as published by 
6    *  the Free Software Foundation.
7    *
8    *  soapUI is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without 
9    *  even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 
10   *  See the GNU Lesser General Public License for more details at gnu.org.
11   */
12  
13  package com.eviware.soapui.tools;
14  
15  import com.eviware.soapui.SoapUI;
16  import com.eviware.soapui.impl.wsdl.WsdlProject;
17  import com.eviware.soapui.impl.wsdl.WsdlTestSuite;
18  import com.eviware.soapui.impl.wsdl.testcase.WsdlTestCase;
19  import com.eviware.soapui.impl.wsdl.teststeps.*;
20  import com.eviware.soapui.model.iface.Attachment;
21  import com.eviware.soapui.model.project.ProjectFactoryRegistry;
22  import com.eviware.soapui.model.propertyexpansion.DefaultPropertyExpansionContext;
23  import com.eviware.soapui.model.propertyexpansion.PropertyExpansionContext;
24  import com.eviware.soapui.model.testsuite.*;
25  import com.eviware.soapui.model.testsuite.Assertable.AssertionStatus;
26  import com.eviware.soapui.model.testsuite.AssertionError;
27  import com.eviware.soapui.model.testsuite.TestRunner.Status;
28  import com.eviware.soapui.model.testsuite.TestStepResult.TestStepStatus;
29  import com.eviware.soapui.model.testsuite.TestSuite.TestSuiteRunType;
30  import com.eviware.soapui.report.JUnitReportCollector;
31  import com.eviware.soapui.support.StringUtils;
32  import com.eviware.soapui.support.Tools;
33  import com.eviware.soapui.support.types.StringToObjectMap;
34  import org.apache.commons.cli.CommandLine;
35  
36  import java.io.File;
37  import java.io.FileOutputStream;
38  import java.io.PrintWriter;
39  import java.io.StringWriter;
40  import java.util.*;
41  
42  /***
43   * Standalone test-runner used from maven-plugin, can also be used from
44   * command-line (see xdocs) or directly from other classes.
45   * <p>
46   * For standalone usage, set the project file (with setProjectFile) and other
47   * desired properties before calling run
48   * </p>
49   * 
50   * @author Ole.Matzura
51   */
52  
53  public class SoapUITestCaseRunner extends AbstractSoapUIRunner implements TestRunListener
54  {
55  	public static final String TITLE = "soapUI " + SoapUI.SOAPUI_VERSION + " TestCase Runner";
56  
57  	private String testSuite;
58  	private String testCase;
59  	private List<WsdlMessageAssertion> assertions = new ArrayList<WsdlMessageAssertion>();
60  	private Map<WsdlMessageAssertion, WsdlTestStepResult> assertionResults = new HashMap<WsdlMessageAssertion, WsdlTestStepResult>();
61  	private List<TestCase> runningTests = new ArrayList<TestCase>();
62  	private List<TestCase> failedTests = new ArrayList<TestCase>();
63  	private String endpoint;
64  	private String domain;
65  	private String password;
66  	private String username;
67  	private String host;
68  
69  	private int testSuiteCount;
70  	private int testCaseCount;
71  	private int testStepCount;
72  	private int testAssertionCount;
73  
74  	private boolean printReport;
75  	private boolean exportAll;
76     private boolean ignoreErrors;
77     private boolean junitReport;
78  	private int exportCount;
79  	private JUnitReportCollector reportCollector;
80  	private String wssPasswordType;
81  	private WsdlProject project;
82  
83  	private String projectPassword;
84  
85  	/***
86  	 * Runs the tests in the specified soapUI project file, see soapUI xdocs for
87  	 * details.
88  	 * 
89  	 * @param args
90  	 * @throws Exception
91  	 */
92  
93  	public static void main(String[] args) throws Exception
94  	{
95  		new SoapUITestCaseRunner().runFromCommandLine(args);
96  	}
97  
98  	protected boolean processCommandLine(CommandLine cmd)
99  	{
100 		if (cmd.hasOption("e"))
101 			setEndpoint(cmd.getOptionValue("e"));
102 
103 		if (cmd.hasOption("s"))
104 			setTestSuite(getCommandLineOptionSubstSpace(cmd, "s"));
105 
106 		if (cmd.hasOption("c"))
107 			setTestCase(getCommandLineOptionSubstSpace(cmd, "c"));
108 
109 		if (cmd.hasOption("u"))
110 			setUsername(cmd.getOptionValue("u"));
111 
112 		if (cmd.hasOption("p"))
113 			setPassword(cmd.getOptionValue("p"));
114 
115 		if (cmd.hasOption("w"))
116 			setWssPasswordType(cmd.getOptionValue("w"));
117 
118 		if (cmd.hasOption("d"))
119 			setDomain(cmd.getOptionValue("d"));
120 
121 		if (cmd.hasOption("h"))
122 			setHost(cmd.getOptionValue("h"));
123 
124 		if (cmd.hasOption("f"))
125 			setOutputFolder(getCommandLineOptionSubstSpace(cmd, "f"));
126 
127 		if (cmd.hasOption("t"))
128 			setSettingsFile(getCommandLineOptionSubstSpace(cmd, "t"));
129 
130 		if (cmd.hasOption("x"))
131 		{
132 			setProjectPassword(cmd.getOptionValue("x"));
133 		}
134 
135 		if (cmd.hasOption("v"))
136 		{
137 			setSoapUISettingsPassword(cmd.getOptionValue("v"));
138 		}
139 
140 		setEnableUI(cmd.hasOption("i"));
141 		setPrintReport(cmd.hasOption("r"));
142 		setExportAll(cmd.hasOption("a"));
143 		setJUnitReport(cmd.hasOption("j"));
144 
145 		return true;
146 	}
147 
148 	public void setProjectPassword(String projectPassword)
149 	{
150 		this.projectPassword = projectPassword;
151 	}
152 
153 	public String getProjectPassword()
154 	{
155 		return projectPassword;
156 	}
157 
158 	protected SoapUIOptions initCommandLineOptions()
159 	{
160 		SoapUIOptions options = new SoapUIOptions("testrunner");
161 		options.addOption("e", true, "Sets the endpoint");
162 		options.addOption("s", true, "Sets the testsuite");
163 		options.addOption("c", true, "Sets the testcase");
164 		options.addOption("u", true, "Sets the username");
165 		options.addOption("p", true, "Sets the password");
166 		options.addOption("w", true, "Sets the WSS password type, either 'Text' or 'Digest'");
167 		options.addOption("i", false, "Enables Swing UI for scripts");
168 		options.addOption("d", true, "Sets the domain");
169 		options.addOption("h", true, "Sets the host");
170 		options.addOption("r", false, "Prints a small summary report");
171 		options.addOption("f", true, "Sets the output folder to export results to");
172 		options.addOption("j", false, "Sets the output to include JUnit XML reports");
173 		options.addOption("a", false, "Turns on exporting of all results");
174 		options.addOption("t", true, "Sets the soapui-settings.xml file to use");
175 		options.addOption("x", true, "Sets project password for decryption if project is encrypted");
176 		options.addOption("v", true, "Sets password for soapui-settings.xml file");
177 		return options;
178 	}
179 
180 	/***
181 	 * Add console appender to groovy log
182 	 */
183 
184 	public void setExportAll(boolean exportAll)
185 	{
186 		this.exportAll = exportAll;
187 	}
188 
189 	public void setJUnitReport(boolean junitReport)
190 	{
191 		this.junitReport = junitReport;
192 		if (junitReport)
193 			reportCollector = new JUnitReportCollector();
194 	}
195 
196 	public SoapUITestCaseRunner()
197 	{
198 		super(SoapUITestCaseRunner.TITLE);
199 	}
200 
201 	public SoapUITestCaseRunner(String title)
202 	{
203 		super(title);
204 	}
205 
206 	/***
207 	 * Controls if a short test summary should be printed after the test runs
208 	 * 
209 	 * @param printReport
210 	 *           a flag controlling if a summary should be printed
211 	 */
212 
213 	public void setPrintReport(boolean printReport)
214 	{
215 		this.printReport = printReport;
216 	}
217 
218    public void setIgnoreError( boolean ignoreErrors )
219 	{
220 		this.ignoreErrors = ignoreErrors;
221 	}
222 
223    /***
224 	 * Sets the host to use by all test-requests, the existing endpoint port and
225 	 * path will be used
226 	 * 
227 	 * @param host
228 	 *           the host to use by all requests
229 	 */
230 
231 	public void setHost(String host)
232 	{
233 		this.host = host;
234 	}
235 
236 	/***
237 	 * Sets the domain to use for any authentications
238 	 * 
239 	 * @param domain
240 	 *           the domain to use for any authentications
241 	 */
242 
243 	public void setDomain(String domain)
244 	{
245 		this.domain = domain;
246 	}
247 
248 	/***
249 	 * Sets the password to use for any authentications
250 	 * 
251 	 * @param password the password to use for any authentications
252 	 */
253 
254 	public void setPassword(String password)
255 	{
256 		this.password = password;
257 	}
258 
259 	/***
260 	 * Sets the WSS password-type to use for any authentications. Setting this
261 	 * will result in the addition of WS-Security UsernamePassword tokens to any
262 	 * outgoing request containing the specified username and password.
263 	 * 
264 	 * @param wssPasswordType
265 	 *           the wss-password type to use, either 'Text' or 'Digest'
266 	 */
267 
268 	public void setWssPasswordType(String wssPasswordType)
269 	{
270 		this.wssPasswordType = wssPasswordType;
271 	}
272 
273 	/***
274 	 * Sets the username to use for any authentications
275 	 * 
276 	 * @param username
277 	 *           the username to use for any authentications
278 	 */
279 
280 	public void setUsername(String username)
281 	{
282 		this.username = username;
283 	}
284 
285 	/***
286 	 * Runs the testcases as configured with setXXX methods
287 	 * 
288 	 * @throws Exception
289 	 *            thrown if any tests fail
290 	 */
291 
292 	public boolean runRunner() throws Exception
293 	{
294 		initGroovyLog();
295 
296 		assertions.clear();
297 
298 		String projectFile = getProjectFile();
299 
300 //		project = new WsdlProject(projectFile, getProjectPassword());
301 		project = (WsdlProject) ProjectFactoryRegistry.getProjectFactory("wsdl").createNew(projectFile, getProjectPassword());
302 		
303 		if (project.isDisabled())
304 			throw new Exception("Failed to load soapUI project file [" + projectFile + "]");
305 
306 		initProject();
307 		ensureOutputFolder( project );
308 		
309 		log.info("Running soapUI tests in project [" + project.getName() + "]");
310 
311 		long startTime = System.nanoTime();
312 
313 		// start by listening to all testcases.. (since one testcase can call
314 		// another)
315 		for (int c = 0; c < project.getTestSuiteCount(); c++)
316 		{
317 			TestSuite suite = project.getTestSuiteAt(c);
318 			for (int i = 0; i < suite.getTestCaseCount(); i++)
319 			{
320 				TestCase tc = suite.getTestCaseAt(i);
321 				addListeners(tc);
322 			}
323 		}
324 
325 		// now run tests..
326 		for (int c = 0; c < project.getTestSuiteCount(); c++)
327 		{
328 			WsdlTestSuite ts = project.getTestSuiteAt(c);
329 			if (!ts.isDisabled() && (testSuite == null || ts.getName().equalsIgnoreCase(testSuite)))
330 			{
331 				runSuite(ts);
332 				testSuiteCount++;
333 
334 				// wait for tests to finish if running in parallell mode
335 				if (!runningTests.isEmpty())
336 					log.info("Waiting for " + runningTests.size() + " tests to finish");
337 
338 				while (!runningTests.isEmpty())
339 				{
340 					Thread.sleep(100);
341 				}
342 			}
343 		}
344 
345 		long timeTaken = (System.nanoTime() - startTime) / 1000000;
346 
347 		if (printReport)
348 		{
349 			printReport(timeTaken);
350 		}
351 		
352 		exportReports( project );
353 
354 		if (assertions.size() > 0 || failedTests.size() > 0 && !ignoreErrors )
355 		{
356 			throwFailureException();
357 		}
358 
359 		return true;
360 	}
361 
362 	protected void initProject() throws Exception
363 	{
364 	}
365 
366 	protected void exportReports( WsdlProject project ) throws Exception
367 	{
368 		if (junitReport)
369 		{
370 			exportJUnitReports(reportCollector, getAbsoluteOutputFolder( project ));
371 		}
372 	}
373 
374 	protected void addListeners(TestCase tc)
375 	{
376 		tc.addTestRunListener(this);
377 		if (junitReport)
378 			tc.addTestRunListener(reportCollector);
379 	}
380 
381 	protected void throwFailureException() throws Exception
382 	{
383 		StringBuffer buf = new StringBuffer();
384 
385 		for (int c = 0; c < assertions.size(); c++)
386 		{
387 			WsdlMessageAssertion assertion = assertions.get(c);
388 			Assertable assertable = assertion.getAssertable();
389 			if (assertable instanceof WsdlTestStep)
390 				failedTests.remove(((WsdlTestStep) assertable).getTestCase());
391 
392 			buf.append(assertion.getName() + " in [" + assertable.getModelItem().getName() + "] failed;\n");
393 			buf.append(Arrays.toString(assertion.getErrors()) + "\n");
394 
395 			WsdlTestStepResult result = assertionResults.get(assertion);
396 			StringWriter stringWriter = new StringWriter();
397 			PrintWriter writer = new PrintWriter(stringWriter);
398 			result.writeTo(writer);
399 			buf.append(stringWriter.toString());
400 		}
401 
402 		while (!failedTests.isEmpty())
403 		{
404 			buf.append("TestCase [" + failedTests.remove(0).getName() + "] failed without assertions\n");
405 		}
406 
407 		throw new Exception(buf.toString());
408 	}
409 
410 	public void exportJUnitReports(JUnitReportCollector collector, String folder) throws Exception
411 	{
412 		collector.saveReports(folder == null ? "" : folder);
413 	}
414 
415 	public void printReport(long timeTaken)
416 	{
417 		System.out.println();
418 		System.out.println("SoapUI " + SoapUI.SOAPUI_VERSION + " TestCaseRunner Summary");
419 		System.out.println("-----------------------------");
420 		System.out.println("Time Taken: " + timeTaken + "ms");
421 		System.out.println("Total TestSuites: " + testSuiteCount);
422 		System.out.println("Total TestCases: " + testCaseCount + " (" + failedTests.size() + " failed)");
423 		System.out.println("Total TestSteps: " + testStepCount);
424 		System.out.println("Total Request Assertions: " + testAssertionCount);
425 		System.out.println("Total Failed Assertions: " + assertions.size());
426 		System.out.println("Total Exported Results: " + exportCount);
427 	}
428 
429 	/***
430 	 * Run tests in the specified TestSuite
431 	 * 
432 	 * @param suite
433 	 *           the TestSuite to run
434 	 */
435 
436 	public void runSuite(WsdlTestSuite suite)
437 	{
438 		log.info(("Running soapUI suite [" + suite.getName() + "], runType = " + suite.getRunType()));
439 		PropertyExpansionContext context = new DefaultPropertyExpansionContext(suite);
440 		long start = System.currentTimeMillis();
441 
442 		try
443 		{
444 			suite.runSetupScript(context);
445 			for (int c = 0; c < suite.getTestCaseCount(); c++)
446 			{
447 				WsdlTestCase tc = suite.getTestCaseAt(c);
448 
449 				String name = tc.getName();
450 				if (testCase == null || name.equalsIgnoreCase(testCase))
451 				{
452 					if (!tc.isDisabled())
453 					{
454 						runTestCase(tc);
455 					}
456 					else
457 					{
458 						log.info("Skipping disabled testcase [" + name + "]");
459 					}
460 				}
461 				else
462 				{
463 					log.info("Skipping testcase [" + name + "], filter is [" + testCase + "]");
464 				}
465 			}
466 		}
467 		catch (Exception e)
468 		{
469 			e.printStackTrace();
470 		}
471 		finally
472 		{
473 			try
474 			{
475 				suite.runTearDownScript(context);
476 			}
477 			catch (Exception e)
478 			{
479 				e.printStackTrace();
480 			}
481 		}
482 
483 		log.info("soapUI suite [" + suite.getName() + "] finished in " + (System.currentTimeMillis() - start) + "ms");
484 	}
485 
486 	/***
487 	 * Runs the specified TestCase
488 	 * 
489 	 * @param testCase
490 	 *           the testcase to run
491 	 */
492 
493 	private void runTestCase(TestCase testCase)
494 	{
495 		runningTests.add(testCase);
496 		testCase.run(new StringToObjectMap(), testCase.getTestSuite().getRunType() == TestSuiteRunType.PARALLEL);
497 	}
498 
499 	/***
500 	 * Sets the testcase to run
501 	 * 
502 	 * @param testCase
503 	 *           the testcase to run
504 	 */
505 
506 	public void setTestCase(String testCase)
507 	{
508 		this.testCase = testCase;
509 	}
510 
511 	/***
512 	 * Sets the endpoint to use for all test requests
513 	 * 
514 	 * @param endpoint
515 	 *           the endpoint to use for all test requests
516 	 */
517 
518 	public void setEndpoint(String endpoint)
519 	{
520 		this.endpoint = endpoint.trim();
521 	}
522 
523 	/***
524 	 * Sets the TestSuite to run. If not set all TestSuites in the specified
525 	 * project file are run
526 	 * 
527 	 * @param testSuite
528 	 *           the testSuite to run.
529 	 */
530 
531 	public void setTestSuite(String testSuite)
532 	{
533 		this.testSuite = testSuite;
534 	}
535 
536 	public void beforeRun(TestRunner testRunner, TestRunContext runContext)
537 	{
538 		log.info("Running soapUI testcase [" + testRunner.getTestCase().getName() + "]");
539 	}
540 
541 	public void beforeStep(TestRunner testRunner, TestRunContext runContext)
542 	{
543 		TestStep currentStep = runContext.getCurrentStep();
544 		log.info("running step [" + currentStep.getName() + "]");
545 
546 		if (currentStep instanceof WsdlTestRequestStep)
547 		{
548 			WsdlTestRequestStep requestStep = (WsdlTestRequestStep) currentStep;
549 			if (endpoint != null && endpoint.length() > 0)
550 			{
551 				requestStep.getTestRequest().setEndpoint(endpoint);
552 			}
553 
554 			if (host != null && host.length() > 0)
555 			{
556 				try
557 				{
558 					String ep = Tools.replaceHost(requestStep.getTestRequest().getEndpoint(), host);
559 					requestStep.getTestRequest().setEndpoint(ep);
560 				}
561 				catch (Exception e)
562 				{
563 					log.error("Failed to set host on endpoint", e);
564 				}
565 			}
566 
567 			if (username != null && username.length() > 0)
568 			{
569 				requestStep.getTestRequest().setUsername(username);
570 			}
571 
572 			if (password != null && password.length() > 0)
573 			{
574 				requestStep.getTestRequest().setPassword(password);
575 			}
576 
577 			if (domain != null && domain.length() > 0)
578 			{
579 				requestStep.getTestRequest().setDomain(domain);
580 			}
581 
582 			if (wssPasswordType != null && wssPasswordType.length() > 0)
583 			{
584 				requestStep.getTestRequest().setWssPasswordType(
585 						wssPasswordType.equals("Digest") ? WsdlTestRequest.PW_TYPE_DIGEST : WsdlTestRequest.PW_TYPE_TEXT);
586 			}
587 		}
588 	}
589 
590 	public void afterStep(TestRunner testRunner, TestRunContext runContext, TestStepResult result)
591 	{
592 		TestStep currentStep = runContext.getCurrentStep();
593 
594 		if (currentStep instanceof WsdlTestRequestStep)
595 		{
596 			WsdlTestRequestStep requestStep = (WsdlTestRequestStep) currentStep;
597 			for (int c = 0; c < requestStep.getAssertionCount(); c++)
598 			{
599 				WsdlMessageAssertion assertion = requestStep.getAssertionAt(c);
600 				log.info("Assertion [" + assertion.getName() + "] has status " + assertion.getStatus());
601 				if (assertion.getStatus() == AssertionStatus.FAILED)
602 				{
603 					for (AssertionError error : assertion.getErrors())
604 						log.error("ASSERTION FAILED -> " + error.getMessage());
605 
606 					assertions.add(assertion);
607 					assertionResults.put(assertion, (WsdlTestStepResult) result);
608 				}
609 
610 				testAssertionCount++;
611 			}
612 		}
613 
614 		String countPropertyName = currentStep.getName() + " run count";
615 		Long count = (Long) runContext.getProperty(countPropertyName);
616 		if (count == null)
617 		{
618 			count = new Long(0);
619 		}
620 
621 		runContext.setProperty(countPropertyName, new Long(count.longValue() + 1));
622 
623 		if (result.getStatus() == TestStepStatus.FAILED || exportAll)
624 		{
625 			try
626 			{
627 				TestCase tc = currentStep.getTestCase();
628 				String nameBase = StringUtils.createFileName(tc.getTestSuite().getName(), '_') + "-"
629 						+ StringUtils.createFileName(tc.getName(), '_') + "-"
630 						+ StringUtils.createFileName(currentStep.getName(), '_') + "-" + count.longValue() + "-"
631 						+ result.getStatus();
632 
633 				String absoluteOutputFolder = getAbsoluteOutputFolder( project );
634 				String fileName = absoluteOutputFolder + File.separator + nameBase + ".txt";
635 
636 				if (result.getStatus() == TestStepStatus.FAILED)
637 					log.error(currentStep.getName() + " failed, exporting to [" + fileName + "]");
638 
639 				PrintWriter writer = new PrintWriter(fileName);
640 				result.writeTo(writer);
641 				writer.close();
642 
643 				// write attachments
644 				if (result instanceof WsdlTestRequestStepResult)
645 				{
646 					Attachment[] attachments = ((WsdlTestRequestStepResult) result).getResponseAttachments();
647 					if (attachments != null && attachments.length > 0)
648 					{
649 						for (int c = 0; c < attachments.length; c++)
650 						{
651 							fileName = nameBase + "-attachment-" + (c + 1) + ".";
652 
653 							Attachment attachment = attachments[c];
654 							String contentType = attachment.getContentType();
655 							if (!"application/octet-stream".equals(contentType) && contentType != null
656 									&& contentType.indexOf('/') != -1)
657 							{
658 								fileName += contentType.substring(contentType.lastIndexOf('/') + 1);
659 							}
660 							else
661 							{
662 								fileName += "dat";
663 							}
664 
665 							fileName = absoluteOutputFolder + File.separator + fileName;
666 
667 							FileOutputStream outFile = new FileOutputStream(fileName);
668 							Tools.writeAll(outFile, attachment.getInputStream());
669 							outFile.close();
670 						}
671 					}
672 				}
673 
674 				exportCount++;
675 			}
676 			catch (Exception e)
677 			{
678 				log.error("Error saving failed result: " + e, e);
679 			}
680 		}
681 
682 		testStepCount++;
683 	}
684 
685 	public void afterRun(TestRunner testRunner, TestRunContext runContext)
686 	{
687 		log.info("Finished running soapUI testcase [" + testRunner.getTestCase().getName() + "], time taken: "
688 				+ testRunner.getTimeTaken() + "ms, status: " + testRunner.getStatus());
689 
690 		if (testRunner.getStatus() == Status.FAILED)
691 		{
692 			failedTests.add(testRunner.getTestCase());
693 		}
694 
695 		runningTests.remove(testRunner.getTestCase());
696 
697 		testCaseCount++;
698 	}
699 
700 	protected WsdlProject getProject()
701 	{
702 		return project;
703 	}
704 }