1
2
3
4
5
6
7
8
9
10
11
12
13 package com.eviware.soapui.tools;
14
15 import com.eviware.soapui.SoapUI;
16 import com.eviware.soapui.impl.wsdl.WsdlProject;
17 import com.eviware.soapui.impl.wsdl.WsdlTestSuite;
18 import com.eviware.soapui.impl.wsdl.testcase.WsdlTestCase;
19 import com.eviware.soapui.impl.wsdl.teststeps.*;
20 import com.eviware.soapui.model.iface.Attachment;
21 import com.eviware.soapui.model.project.ProjectFactoryRegistry;
22 import com.eviware.soapui.model.propertyexpansion.DefaultPropertyExpansionContext;
23 import com.eviware.soapui.model.propertyexpansion.PropertyExpansionContext;
24 import com.eviware.soapui.model.testsuite.*;
25 import com.eviware.soapui.model.testsuite.Assertable.AssertionStatus;
26 import com.eviware.soapui.model.testsuite.AssertionError;
27 import com.eviware.soapui.model.testsuite.TestRunner.Status;
28 import com.eviware.soapui.model.testsuite.TestStepResult.TestStepStatus;
29 import com.eviware.soapui.model.testsuite.TestSuite.TestSuiteRunType;
30 import com.eviware.soapui.report.JUnitReportCollector;
31 import com.eviware.soapui.support.StringUtils;
32 import com.eviware.soapui.support.Tools;
33 import com.eviware.soapui.support.types.StringToObjectMap;
34 import org.apache.commons.cli.CommandLine;
35
36 import java.io.File;
37 import java.io.FileOutputStream;
38 import java.io.PrintWriter;
39 import java.io.StringWriter;
40 import java.util.*;
41
42 /***
43 * Standalone test-runner used from maven-plugin, can also be used from
44 * command-line (see xdocs) or directly from other classes.
45 * <p>
46 * For standalone usage, set the project file (with setProjectFile) and other
47 * desired properties before calling run
48 * </p>
49 *
50 * @author Ole.Matzura
51 */
52
53 public class SoapUITestCaseRunner extends AbstractSoapUIRunner implements TestRunListener
54 {
55 public static final String TITLE = "soapUI " + SoapUI.SOAPUI_VERSION + " TestCase Runner";
56
57 private String testSuite;
58 private String testCase;
59 private List<WsdlMessageAssertion> assertions = new ArrayList<WsdlMessageAssertion>();
60 private Map<WsdlMessageAssertion, WsdlTestStepResult> assertionResults = new HashMap<WsdlMessageAssertion, WsdlTestStepResult>();
61 private List<TestCase> runningTests = new ArrayList<TestCase>();
62 private List<TestCase> failedTests = new ArrayList<TestCase>();
63 private String endpoint;
64 private String domain;
65 private String password;
66 private String username;
67 private String host;
68
69 private int testSuiteCount;
70 private int testCaseCount;
71 private int testStepCount;
72 private int testAssertionCount;
73
74 private boolean printReport;
75 private boolean exportAll;
76 private boolean ignoreErrors;
77 private boolean junitReport;
78 private int exportCount;
79 private JUnitReportCollector reportCollector;
80 private String wssPasswordType;
81 private WsdlProject project;
82
83 private String projectPassword;
84
85 /***
86 * Runs the tests in the specified soapUI project file, see soapUI xdocs for
87 * details.
88 *
89 * @param args
90 * @throws Exception
91 */
92
93 public static void main(String[] args) throws Exception
94 {
95 new SoapUITestCaseRunner().runFromCommandLine(args);
96 }
97
98 protected boolean processCommandLine(CommandLine cmd)
99 {
100 if (cmd.hasOption("e"))
101 setEndpoint(cmd.getOptionValue("e"));
102
103 if (cmd.hasOption("s"))
104 setTestSuite(getCommandLineOptionSubstSpace(cmd, "s"));
105
106 if (cmd.hasOption("c"))
107 setTestCase(getCommandLineOptionSubstSpace(cmd, "c"));
108
109 if (cmd.hasOption("u"))
110 setUsername(cmd.getOptionValue("u"));
111
112 if (cmd.hasOption("p"))
113 setPassword(cmd.getOptionValue("p"));
114
115 if (cmd.hasOption("w"))
116 setWssPasswordType(cmd.getOptionValue("w"));
117
118 if (cmd.hasOption("d"))
119 setDomain(cmd.getOptionValue("d"));
120
121 if (cmd.hasOption("h"))
122 setHost(cmd.getOptionValue("h"));
123
124 if (cmd.hasOption("f"))
125 setOutputFolder(getCommandLineOptionSubstSpace(cmd, "f"));
126
127 if (cmd.hasOption("t"))
128 setSettingsFile(getCommandLineOptionSubstSpace(cmd, "t"));
129
130 if (cmd.hasOption("x"))
131 {
132 setProjectPassword(cmd.getOptionValue("x"));
133 }
134
135 if (cmd.hasOption("v"))
136 {
137 setSoapUISettingsPassword(cmd.getOptionValue("v"));
138 }
139
140 setEnableUI(cmd.hasOption("i"));
141 setPrintReport(cmd.hasOption("r"));
142 setExportAll(cmd.hasOption("a"));
143 setJUnitReport(cmd.hasOption("j"));
144
145 return true;
146 }
147
148 public void setProjectPassword(String projectPassword)
149 {
150 this.projectPassword = projectPassword;
151 }
152
153 public String getProjectPassword()
154 {
155 return projectPassword;
156 }
157
158 protected SoapUIOptions initCommandLineOptions()
159 {
160 SoapUIOptions options = new SoapUIOptions("testrunner");
161 options.addOption("e", true, "Sets the endpoint");
162 options.addOption("s", true, "Sets the testsuite");
163 options.addOption("c", true, "Sets the testcase");
164 options.addOption("u", true, "Sets the username");
165 options.addOption("p", true, "Sets the password");
166 options.addOption("w", true, "Sets the WSS password type, either 'Text' or 'Digest'");
167 options.addOption("i", false, "Enables Swing UI for scripts");
168 options.addOption("d", true, "Sets the domain");
169 options.addOption("h", true, "Sets the host");
170 options.addOption("r", false, "Prints a small summary report");
171 options.addOption("f", true, "Sets the output folder to export results to");
172 options.addOption("j", false, "Sets the output to include JUnit XML reports");
173 options.addOption("a", false, "Turns on exporting of all results");
174 options.addOption("t", true, "Sets the soapui-settings.xml file to use");
175 options.addOption("x", true, "Sets project password for decryption if project is encrypted");
176 options.addOption("v", true, "Sets password for soapui-settings.xml file");
177 return options;
178 }
179
180 /***
181 * Add console appender to groovy log
182 */
183
184 public void setExportAll(boolean exportAll)
185 {
186 this.exportAll = exportAll;
187 }
188
189 public void setJUnitReport(boolean junitReport)
190 {
191 this.junitReport = junitReport;
192 if (junitReport)
193 reportCollector = new JUnitReportCollector();
194 }
195
196 public SoapUITestCaseRunner()
197 {
198 super(SoapUITestCaseRunner.TITLE);
199 }
200
201 public SoapUITestCaseRunner(String title)
202 {
203 super(title);
204 }
205
206 /***
207 * Controls if a short test summary should be printed after the test runs
208 *
209 * @param printReport
210 * a flag controlling if a summary should be printed
211 */
212
213 public void setPrintReport(boolean printReport)
214 {
215 this.printReport = printReport;
216 }
217
218 public void setIgnoreError( boolean ignoreErrors )
219 {
220 this.ignoreErrors = ignoreErrors;
221 }
222
223 /***
224 * Sets the host to use by all test-requests, the existing endpoint port and
225 * path will be used
226 *
227 * @param host
228 * the host to use by all requests
229 */
230
231 public void setHost(String host)
232 {
233 this.host = host;
234 }
235
236 /***
237 * Sets the domain to use for any authentications
238 *
239 * @param domain
240 * the domain to use for any authentications
241 */
242
243 public void setDomain(String domain)
244 {
245 this.domain = domain;
246 }
247
248 /***
249 * Sets the password to use for any authentications
250 *
251 * @param password the password to use for any authentications
252 */
253
254 public void setPassword(String password)
255 {
256 this.password = password;
257 }
258
259 /***
260 * Sets the WSS password-type to use for any authentications. Setting this
261 * will result in the addition of WS-Security UsernamePassword tokens to any
262 * outgoing request containing the specified username and password.
263 *
264 * @param wssPasswordType
265 * the wss-password type to use, either 'Text' or 'Digest'
266 */
267
268 public void setWssPasswordType(String wssPasswordType)
269 {
270 this.wssPasswordType = wssPasswordType;
271 }
272
273 /***
274 * Sets the username to use for any authentications
275 *
276 * @param username
277 * the username to use for any authentications
278 */
279
280 public void setUsername(String username)
281 {
282 this.username = username;
283 }
284
285 /***
286 * Runs the testcases as configured with setXXX methods
287 *
288 * @throws Exception
289 * thrown if any tests fail
290 */
291
292 public boolean runRunner() throws Exception
293 {
294 initGroovyLog();
295
296 assertions.clear();
297
298 String projectFile = getProjectFile();
299
300
301 project = (WsdlProject) ProjectFactoryRegistry.getProjectFactory("wsdl").createNew(projectFile, getProjectPassword());
302
303 if (project.isDisabled())
304 throw new Exception("Failed to load soapUI project file [" + projectFile + "]");
305
306 initProject();
307 ensureOutputFolder( project );
308
309 log.info("Running soapUI tests in project [" + project.getName() + "]");
310
311 long startTime = System.nanoTime();
312
313
314
315 for (int c = 0; c < project.getTestSuiteCount(); c++)
316 {
317 TestSuite suite = project.getTestSuiteAt(c);
318 for (int i = 0; i < suite.getTestCaseCount(); i++)
319 {
320 TestCase tc = suite.getTestCaseAt(i);
321 addListeners(tc);
322 }
323 }
324
325
326 for (int c = 0; c < project.getTestSuiteCount(); c++)
327 {
328 WsdlTestSuite ts = project.getTestSuiteAt(c);
329 if (!ts.isDisabled() && (testSuite == null || ts.getName().equalsIgnoreCase(testSuite)))
330 {
331 runSuite(ts);
332 testSuiteCount++;
333
334
335 if (!runningTests.isEmpty())
336 log.info("Waiting for " + runningTests.size() + " tests to finish");
337
338 while (!runningTests.isEmpty())
339 {
340 Thread.sleep(100);
341 }
342 }
343 }
344
345 long timeTaken = (System.nanoTime() - startTime) / 1000000;
346
347 if (printReport)
348 {
349 printReport(timeTaken);
350 }
351
352 exportReports( project );
353
354 if (assertions.size() > 0 || failedTests.size() > 0 && !ignoreErrors )
355 {
356 throwFailureException();
357 }
358
359 return true;
360 }
361
362 protected void initProject() throws Exception
363 {
364 }
365
366 protected void exportReports( WsdlProject project ) throws Exception
367 {
368 if (junitReport)
369 {
370 exportJUnitReports(reportCollector, getAbsoluteOutputFolder( project ));
371 }
372 }
373
374 protected void addListeners(TestCase tc)
375 {
376 tc.addTestRunListener(this);
377 if (junitReport)
378 tc.addTestRunListener(reportCollector);
379 }
380
381 protected void throwFailureException() throws Exception
382 {
383 StringBuffer buf = new StringBuffer();
384
385 for (int c = 0; c < assertions.size(); c++)
386 {
387 WsdlMessageAssertion assertion = assertions.get(c);
388 Assertable assertable = assertion.getAssertable();
389 if (assertable instanceof WsdlTestStep)
390 failedTests.remove(((WsdlTestStep) assertable).getTestCase());
391
392 buf.append(assertion.getName() + " in [" + assertable.getModelItem().getName() + "] failed;\n");
393 buf.append(Arrays.toString(assertion.getErrors()) + "\n");
394
395 WsdlTestStepResult result = assertionResults.get(assertion);
396 StringWriter stringWriter = new StringWriter();
397 PrintWriter writer = new PrintWriter(stringWriter);
398 result.writeTo(writer);
399 buf.append(stringWriter.toString());
400 }
401
402 while (!failedTests.isEmpty())
403 {
404 buf.append("TestCase [" + failedTests.remove(0).getName() + "] failed without assertions\n");
405 }
406
407 throw new Exception(buf.toString());
408 }
409
410 public void exportJUnitReports(JUnitReportCollector collector, String folder) throws Exception
411 {
412 collector.saveReports(folder == null ? "" : folder);
413 }
414
415 public void printReport(long timeTaken)
416 {
417 System.out.println();
418 System.out.println("SoapUI " + SoapUI.SOAPUI_VERSION + " TestCaseRunner Summary");
419 System.out.println("-----------------------------");
420 System.out.println("Time Taken: " + timeTaken + "ms");
421 System.out.println("Total TestSuites: " + testSuiteCount);
422 System.out.println("Total TestCases: " + testCaseCount + " (" + failedTests.size() + " failed)");
423 System.out.println("Total TestSteps: " + testStepCount);
424 System.out.println("Total Request Assertions: " + testAssertionCount);
425 System.out.println("Total Failed Assertions: " + assertions.size());
426 System.out.println("Total Exported Results: " + exportCount);
427 }
428
429 /***
430 * Run tests in the specified TestSuite
431 *
432 * @param suite
433 * the TestSuite to run
434 */
435
436 public void runSuite(WsdlTestSuite suite)
437 {
438 log.info(("Running soapUI suite [" + suite.getName() + "], runType = " + suite.getRunType()));
439 PropertyExpansionContext context = new DefaultPropertyExpansionContext(suite);
440 long start = System.currentTimeMillis();
441
442 try
443 {
444 suite.runSetupScript(context);
445 for (int c = 0; c < suite.getTestCaseCount(); c++)
446 {
447 WsdlTestCase tc = suite.getTestCaseAt(c);
448
449 String name = tc.getName();
450 if (testCase == null || name.equalsIgnoreCase(testCase))
451 {
452 if (!tc.isDisabled())
453 {
454 runTestCase(tc);
455 }
456 else
457 {
458 log.info("Skipping disabled testcase [" + name + "]");
459 }
460 }
461 else
462 {
463 log.info("Skipping testcase [" + name + "], filter is [" + testCase + "]");
464 }
465 }
466 }
467 catch (Exception e)
468 {
469 e.printStackTrace();
470 }
471 finally
472 {
473 try
474 {
475 suite.runTearDownScript(context);
476 }
477 catch (Exception e)
478 {
479 e.printStackTrace();
480 }
481 }
482
483 log.info("soapUI suite [" + suite.getName() + "] finished in " + (System.currentTimeMillis() - start) + "ms");
484 }
485
486 /***
487 * Runs the specified TestCase
488 *
489 * @param testCase
490 * the testcase to run
491 */
492
493 private void runTestCase(TestCase testCase)
494 {
495 runningTests.add(testCase);
496 testCase.run(new StringToObjectMap(), testCase.getTestSuite().getRunType() == TestSuiteRunType.PARALLEL);
497 }
498
499 /***
500 * Sets the testcase to run
501 *
502 * @param testCase
503 * the testcase to run
504 */
505
506 public void setTestCase(String testCase)
507 {
508 this.testCase = testCase;
509 }
510
511 /***
512 * Sets the endpoint to use for all test requests
513 *
514 * @param endpoint
515 * the endpoint to use for all test requests
516 */
517
518 public void setEndpoint(String endpoint)
519 {
520 this.endpoint = endpoint.trim();
521 }
522
523 /***
524 * Sets the TestSuite to run. If not set all TestSuites in the specified
525 * project file are run
526 *
527 * @param testSuite
528 * the testSuite to run.
529 */
530
531 public void setTestSuite(String testSuite)
532 {
533 this.testSuite = testSuite;
534 }
535
536 public void beforeRun(TestRunner testRunner, TestRunContext runContext)
537 {
538 log.info("Running soapUI testcase [" + testRunner.getTestCase().getName() + "]");
539 }
540
541 public void beforeStep(TestRunner testRunner, TestRunContext runContext)
542 {
543 TestStep currentStep = runContext.getCurrentStep();
544 log.info("running step [" + currentStep.getName() + "]");
545
546 if (currentStep instanceof WsdlTestRequestStep)
547 {
548 WsdlTestRequestStep requestStep = (WsdlTestRequestStep) currentStep;
549 if (endpoint != null && endpoint.length() > 0)
550 {
551 requestStep.getTestRequest().setEndpoint(endpoint);
552 }
553
554 if (host != null && host.length() > 0)
555 {
556 try
557 {
558 String ep = Tools.replaceHost(requestStep.getTestRequest().getEndpoint(), host);
559 requestStep.getTestRequest().setEndpoint(ep);
560 }
561 catch (Exception e)
562 {
563 log.error("Failed to set host on endpoint", e);
564 }
565 }
566
567 if (username != null && username.length() > 0)
568 {
569 requestStep.getTestRequest().setUsername(username);
570 }
571
572 if (password != null && password.length() > 0)
573 {
574 requestStep.getTestRequest().setPassword(password);
575 }
576
577 if (domain != null && domain.length() > 0)
578 {
579 requestStep.getTestRequest().setDomain(domain);
580 }
581
582 if (wssPasswordType != null && wssPasswordType.length() > 0)
583 {
584 requestStep.getTestRequest().setWssPasswordType(
585 wssPasswordType.equals("Digest") ? WsdlTestRequest.PW_TYPE_DIGEST : WsdlTestRequest.PW_TYPE_TEXT);
586 }
587 }
588 }
589
590 public void afterStep(TestRunner testRunner, TestRunContext runContext, TestStepResult result)
591 {
592 TestStep currentStep = runContext.getCurrentStep();
593
594 if (currentStep instanceof WsdlTestRequestStep)
595 {
596 WsdlTestRequestStep requestStep = (WsdlTestRequestStep) currentStep;
597 for (int c = 0; c < requestStep.getAssertionCount(); c++)
598 {
599 WsdlMessageAssertion assertion = requestStep.getAssertionAt(c);
600 log.info("Assertion [" + assertion.getName() + "] has status " + assertion.getStatus());
601 if (assertion.getStatus() == AssertionStatus.FAILED)
602 {
603 for (AssertionError error : assertion.getErrors())
604 log.error("ASSERTION FAILED -> " + error.getMessage());
605
606 assertions.add(assertion);
607 assertionResults.put(assertion, (WsdlTestStepResult) result);
608 }
609
610 testAssertionCount++;
611 }
612 }
613
614 String countPropertyName = currentStep.getName() + " run count";
615 Long count = (Long) runContext.getProperty(countPropertyName);
616 if (count == null)
617 {
618 count = new Long(0);
619 }
620
621 runContext.setProperty(countPropertyName, new Long(count.longValue() + 1));
622
623 if (result.getStatus() == TestStepStatus.FAILED || exportAll)
624 {
625 try
626 {
627 TestCase tc = currentStep.getTestCase();
628 String nameBase = StringUtils.createFileName(tc.getTestSuite().getName(), '_') + "-"
629 + StringUtils.createFileName(tc.getName(), '_') + "-"
630 + StringUtils.createFileName(currentStep.getName(), '_') + "-" + count.longValue() + "-"
631 + result.getStatus();
632
633 String absoluteOutputFolder = getAbsoluteOutputFolder( project );
634 String fileName = absoluteOutputFolder + File.separator + nameBase + ".txt";
635
636 if (result.getStatus() == TestStepStatus.FAILED)
637 log.error(currentStep.getName() + " failed, exporting to [" + fileName + "]");
638
639 PrintWriter writer = new PrintWriter(fileName);
640 result.writeTo(writer);
641 writer.close();
642
643
644 if (result instanceof WsdlTestRequestStepResult)
645 {
646 Attachment[] attachments = ((WsdlTestRequestStepResult) result).getResponseAttachments();
647 if (attachments != null && attachments.length > 0)
648 {
649 for (int c = 0; c < attachments.length; c++)
650 {
651 fileName = nameBase + "-attachment-" + (c + 1) + ".";
652
653 Attachment attachment = attachments[c];
654 String contentType = attachment.getContentType();
655 if (!"application/octet-stream".equals(contentType) && contentType != null
656 && contentType.indexOf('/') != -1)
657 {
658 fileName += contentType.substring(contentType.lastIndexOf('/') + 1);
659 }
660 else
661 {
662 fileName += "dat";
663 }
664
665 fileName = absoluteOutputFolder + File.separator + fileName;
666
667 FileOutputStream outFile = new FileOutputStream(fileName);
668 Tools.writeAll(outFile, attachment.getInputStream());
669 outFile.close();
670 }
671 }
672 }
673
674 exportCount++;
675 }
676 catch (Exception e)
677 {
678 log.error("Error saving failed result: " + e, e);
679 }
680 }
681
682 testStepCount++;
683 }
684
685 public void afterRun(TestRunner testRunner, TestRunContext runContext)
686 {
687 log.info("Finished running soapUI testcase [" + testRunner.getTestCase().getName() + "], time taken: "
688 + testRunner.getTimeTaken() + "ms, status: " + testRunner.getStatus());
689
690 if (testRunner.getStatus() == Status.FAILED)
691 {
692 failedTests.add(testRunner.getTestCase());
693 }
694
695 runningTests.remove(testRunner.getTestCase());
696
697 testCaseCount++;
698 }
699
700 protected WsdlProject getProject()
701 {
702 return project;
703 }
704 }