View Javadoc

1   /*
2    *  soapUI, copyright (C) 2004-2007 eviware.com 
3    *
4    *  soapUI is free software; you can redistribute it and/or modify it under the 
5    *  terms of version 2.1 of the GNU Lesser General Public License as published by 
6    *  the Free Software Foundation.
7    *
8    *  soapUI is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without 
9    *  even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 
10   *  See the GNU Lesser General Public License for more details at gnu.org.
11   */
12  
13  package com.eviware.soapui.impl.wsdl.loadtest.data;
14  
15  import java.beans.PropertyChangeEvent;
16  import java.beans.PropertyChangeListener;
17  import java.util.HashMap;
18  import java.util.List;
19  import java.util.Map;
20  import java.util.Stack;
21  
22  import javax.swing.table.AbstractTableModel;
23  
24  import org.apache.log4j.Logger;
25  
26  import com.eviware.soapui.SoapUI;
27  import com.eviware.soapui.impl.wsdl.loadtest.ColorPalette;
28  import com.eviware.soapui.impl.wsdl.loadtest.WsdlLoadTest;
29  import com.eviware.soapui.impl.wsdl.testcase.WsdlTestCase;
30  import com.eviware.soapui.impl.wsdl.teststeps.WsdlTestStep;
31  import com.eviware.soapui.model.support.LoadTestRunListenerAdapter;
32  import com.eviware.soapui.model.support.TestSuiteListenerAdapter;
33  import com.eviware.soapui.model.testsuite.LoadTestRunContext;
34  import com.eviware.soapui.model.testsuite.LoadTestRunner;
35  import com.eviware.soapui.model.testsuite.TestCase;
36  import com.eviware.soapui.model.testsuite.TestRunContext;
37  import com.eviware.soapui.model.testsuite.TestRunner;
38  import com.eviware.soapui.model.testsuite.TestStep;
39  import com.eviware.soapui.model.testsuite.TestStepResult;
40  
41  /***
42   * Model holding statistics.. should be refactored into interface for different statistic models
43   * 
44   * @author Ole.Matzura
45   */
46  
47  public final class LoadTestStatistics extends AbstractTableModel implements Runnable
48  {
49  	private final static Logger log = Logger.getLogger(LoadTestStatistics.class);
50  	
51  	private final WsdlLoadTest loadTest;
52  	private long [][] data;
53  	
54  	private final static int MIN_COLUMN = 0;
55  	private final static int MAX_COLUMN = 1;
56  	private final static int AVG_COLUMN = 2;
57  	private final static int LAST_COLUMN = 3;
58  	private final static int CNT_COLUMN = 4;
59  	private final static int TPS_COLUMN = 5;
60  	private final static int BYTES_COLUMN = 6;
61  	private final static int BPS_COLUMN = 7;
62  	private final static int ERR_COLUMN = 8;
63  	private final static int SUM_COLUMN = 9;
64  	private final static int CURRENT_CNT_COLUMN = 10;
65  	
66  	public static final int TOTAL = -1;
67  	
68  	public static final int DEFAULT_SAMPLE_INTERVAL = 250;
69  	
70  	private InternalTestRunListener testRunListener;
71  	private InternalTestSuiteListener testSuiteListener;
72  	private InternalPropertyChangeListener propertyChangeListener;
73  	
74  	private StatisticsHistory history;
75  	
76  	private boolean changed;
77  	private long updateFrequency = DEFAULT_SAMPLE_INTERVAL;
78  	private Stack<SamplesHolder> samplesStack = new Stack<SamplesHolder>();
79  	private long currentThreadCountStartTime;
80  	private long totalAverageSum;
81  	private boolean resetStatistics;
82  	private boolean running;
83  	private boolean adding;
84  
85  	public LoadTestStatistics( WsdlLoadTest loadTest )
86  	{
87  		this.loadTest = loadTest;
88  		
89  		testRunListener = new InternalTestRunListener();
90  		testSuiteListener = new InternalTestSuiteListener();
91  		propertyChangeListener = new InternalPropertyChangeListener();
92  		
93  		WsdlTestCase testCase = loadTest.getTestCase();
94  		loadTest.addPropertyChangeListener( propertyChangeListener );
95  		loadTest.addLoadTestRunListener( testRunListener );
96  		testCase.getTestSuite().addTestSuiteListener( testSuiteListener );
97  
98  		for( TestStep testStep : testCase.getTestStepList() )
99  		{
100 			testStep.addPropertyChangeListener( propertyChangeListener );
101 		}
102 
103 		history = new StatisticsHistory( this );
104 		
105 		init();
106 	}
107 
108 	private void init()
109 	{
110 		data = new long[getRowCount()][11];
111 	}
112 	
113 	public StatisticsHistory getHistory()
114 	{
115 		return history;
116 	}
117 
118 	public int getRowCount()
119 	{
120 		return loadTest.getTestCase().getTestStepCount() + 1;
121 	}
122 
123 	public WsdlLoadTest getLoadTest()
124 	{
125 		return loadTest;
126 	}
127 	
128 	public int getColumnCount()
129 	{
130 		return 11;
131 	}
132 
133 	public String getColumnName(int columnIndex)
134 	{
135 		switch( columnIndex )
136 		{
137 			case 0 : return " ";
138 			case 1 : return "Test Step";
139 			case 2 : return Statistic.MININMUM.getName();
140 			case 3 : return Statistic.MAXIMUM.getName();
141 			case 4 : return Statistic.AVERAGE.getName();
142 			case 5 : return Statistic.LAST.getName();
143 			case 6 : return Statistic.COUNT.getName();
144 			case 7 : return Statistic.TPS.getName();
145 			case 8 : return Statistic.BYTES.getName();
146 			case 9 : return Statistic.BPS.getName();
147 			case 10 : return Statistic.ERRORS.getName();
148 		}
149 		return null;
150 	}
151 
152 	public Class<?> getColumnClass(int columnIndex)
153 	{
154 		return String.class;
155 	}
156 
157 	public boolean isCellEditable(int rowIndex, int columnIndex)
158 	{
159 		return false;
160 	}
161 
162 	public long getStatistic( int stepIndex, Statistic statistic )
163 	{
164 		if( stepIndex == TOTAL )
165 			stepIndex = data.length-1;
166 				
167 		if( statistic == Statistic.TPS || statistic == Statistic.AVERAGE )
168 			return data[stepIndex][statistic.getIndex()]/100;
169 		else
170 			return data[stepIndex][statistic.getIndex()];
171 	}
172 	
173 	public Object getValueAt(int rowIndex, int columnIndex)
174 	{
175 		WsdlTestCase testCase = loadTest.getTestCase();
176 		
177 		switch( columnIndex )
178 		{
179 			case 0 : return rowIndex == testCase.getTestStepCount() ? "" : 
180 				 ColorPalette.getColor( testCase.getTestStepAt( rowIndex ) );
181 			case 1 :
182 			{
183 				if( rowIndex == testCase.getTestStepCount() )
184 				{
185 					return "TestCase:";
186 				}
187 				else
188 				{
189 					WsdlTestStep testStep = testCase.getTestStepAt( rowIndex );
190 					if( testStep.isDisabled() )
191 						return testStep.getName() + " (Disabled)";
192 					else
193 						return testStep.getName();
194 				}
195 			}
196 			case 4 :
197 			case 7 : return Float.toString( (float)data[rowIndex][columnIndex-2]/100 );
198 			default :
199 			{
200 				return data == null || rowIndex >= data.length ? "0" : data[rowIndex][columnIndex-2];
201 			}
202 		}
203 	}
204 
205 	public void pushSamples( long[] samples, long[] sizes, long[] sampleCounts, long startTime, long timeTaken )
206 	{
207 		if( samples.length == 0 || sizes.length == 0 )
208 			return;
209 		
210 		samplesStack.push( new SamplesHolder( samples, sizes, sampleCounts, startTime, timeTaken ));
211 	}
212 	
213 	public void run()
214 	{
215 		while( running || !samplesStack.isEmpty() )
216 		{
217 			try
218 			{
219 				while (!samplesStack.isEmpty())
220 				{
221 					SamplesHolder holder = samplesStack.pop();
222 					if (holder != null)
223 						addSamples(holder);
224 				}
225 				
226 				Thread.sleep( 200 );
227 			}
228 			catch (Exception e)
229 			{
230 				SoapUI.logError( e );
231 			}			
232 		}
233 	}
234 	
235 	private void addSamples( SamplesHolder holder )
236 	{
237 		if( adding )
238 			throw new RuntimeException( "Already adding!" );
239 		
240 		adding = true;
241 		
242 		int totalIndex = data.length-1;
243 		if( holder.samples.length != totalIndex || holder.sizes.length != totalIndex )
244 		{
245 			adding = false;
246 			throw new RuntimeException( "Unexpected number of samples: " + holder.samples.length + 
247 					", exptected " + (totalIndex) );
248 		}
249 
250 		
251 		// discard "old" results
252 		if( holder.startTime < currentThreadCountStartTime )
253 		{
254 			adding = false;
255 			return;
256 		}
257 		
258 		//	 first check that this is not a
259 		long timePassed = (holder.startTime+holder.timeTaken)-currentThreadCountStartTime;
260 		
261 		if( resetStatistics )
262 		{
263 			for( int c = 0; c < data.length; c++ )
264 			{
265 				data[c][CURRENT_CNT_COLUMN] = 0;
266 				data[c][AVG_COLUMN] = 0;
267 				data[c][SUM_COLUMN] = 0;
268 				data[c][TPS_COLUMN] = 0;
269 				data[c][BYTES_COLUMN] = 0;
270 			}
271 
272 			totalAverageSum = 0;
273 			resetStatistics = false;
274 		}
275 		
276 		long totalMin = 0;
277 		long totalMax = 0;
278 		long totalBytes = 0;
279 		long totalAvg = 0;
280 		long totalSum = 0;
281 		long totalLast = 0;
282 
283 		long threadCount = loadTest.getThreadCount();
284 		
285 		for( int c = 0; c < holder.samples.length; c++ )
286 		{
287 			if( holder.sampleCounts[c] > 0 )
288 			{
289 				long sampleAvg = holder.samples[c] / holder.sampleCounts[c];
290 				
291 				data[c][LAST_COLUMN] = sampleAvg; 
292 				data[c][CNT_COLUMN] += holder.sampleCounts[c];
293 				data[c][CURRENT_CNT_COLUMN] += holder.sampleCounts[c];
294 				data[c][SUM_COLUMN] += holder.samples[c];
295 				
296 				if( sampleAvg > 0 && (sampleAvg < data[c][MIN_COLUMN] || data[c][CNT_COLUMN] == 1) ) 
297 					data[c][MIN_COLUMN] = sampleAvg;
298 				
299 				if( sampleAvg > data[c][MAX_COLUMN] ) 
300 					data[c][MAX_COLUMN] = sampleAvg;
301 				
302 				float average = (float)data[c][SUM_COLUMN]/(float)data[c][CURRENT_CNT_COLUMN];
303 				
304 				data[c][AVG_COLUMN] = (long) (average*100);
305 				data[c][BYTES_COLUMN] += holder.sizes[c];
306 	
307 				if( timePassed > 0 )
308 				{
309 //					data[c][BPS_COLUMN] = (data[c][BYTES_COLUMN]*1000) / timePassed; 
310 					
311 					if( loadTest.getCalculateTPSOnTimePassed() )
312 					{
313 						data[c][TPS_COLUMN] = (data[c][CURRENT_CNT_COLUMN]*100000) / timePassed;
314 						data[c][BPS_COLUMN] = (data[c][BYTES_COLUMN]*1000) / timePassed; 
315 					}
316 					else
317 					{
318 						data[c][TPS_COLUMN] = (long) (data[c][AVG_COLUMN] > 0 ? 
319 								(100000F/average)*threadCount : 0);
320 						
321 //						long v = (data[c][TPS_COLUMN] * timePassed)/10000000;
322 //						data[c][BPS_COLUMN] = v == 0 ? 0 : data[c][BYTES_COLUMN] / v; 
323 						
324 						long avgBytes = data[c][CNT_COLUMN] == 0 ? 0 : data[c][BYTES_COLUMN] / data[c][CNT_COLUMN];
325 						data[c][BPS_COLUMN] = (avgBytes * data[c][TPS_COLUMN]) / 100;
326 					}
327 				}
328 				
329 				totalMin += data[c][MIN_COLUMN] * holder.sampleCounts[c];
330 				totalMax += data[c][MAX_COLUMN] * holder.sampleCounts[c];
331 				totalBytes += data[c][BYTES_COLUMN] * holder.sampleCounts[c];
332 				totalAvg += data[c][AVG_COLUMN] * holder.sampleCounts[c];
333 				totalSum += data[c][SUM_COLUMN] * holder.sampleCounts[c];
334 				totalLast += data[c][LAST_COLUMN] * holder.sampleCounts[c];
335 			}
336 			else
337 			{
338 				totalMin += data[c][MIN_COLUMN];
339 				totalMax += data[c][MAX_COLUMN];
340 				totalBytes += data[c][BYTES_COLUMN];
341 			}
342 		}
343 		
344 		data[totalIndex][CNT_COLUMN]++;
345 		data[totalIndex][CURRENT_CNT_COLUMN]++;
346 		
347 		totalAverageSum += totalLast*100;
348 		data[totalIndex][AVG_COLUMN] = (long)((float)totalAverageSum / (float)data[totalIndex][CURRENT_CNT_COLUMN]);
349 		data[totalIndex][BYTES_COLUMN] = totalBytes;
350 		
351 		if( timePassed > 0 )
352 		{
353 			if( loadTest.getCalculateTPSOnTimePassed() )
354 			{
355 				data[totalIndex][TPS_COLUMN] = (data[totalIndex][CURRENT_CNT_COLUMN]*100000) / timePassed;
356 				data[totalIndex][BPS_COLUMN] = (data[totalIndex][BYTES_COLUMN]*1000) / timePassed; 
357 			}
358 			else
359 			{
360 				data[totalIndex][TPS_COLUMN] = (long) (data[totalIndex][AVG_COLUMN] > 0 ? 
361 						(10000000F/data[totalIndex][AVG_COLUMN])*threadCount : 0);
362 				
363 				long avgBytes = data[totalIndex][CNT_COLUMN] == 0 ? 0 : 
364 					data[totalIndex][BYTES_COLUMN] / data[totalIndex][CNT_COLUMN];
365 				
366 				data[totalIndex][BPS_COLUMN] = (avgBytes * data[totalIndex][TPS_COLUMN]) / 100;
367 			}
368 		}
369 		
370 		data[totalIndex][MIN_COLUMN] = totalMin;
371 		data[totalIndex][MAX_COLUMN] = totalMax;
372 		data[totalIndex][SUM_COLUMN] = totalSum;
373 		data[totalIndex][LAST_COLUMN] = totalLast;
374 		
375 		if( updateFrequency == 0 )
376 			fireTableDataChanged();
377 		else
378 			changed = true;
379 		
380 		adding = false;
381 	}
382 
383 	private final class Updater implements Runnable
384 	{
385 		public void run()
386 		{
387 			// check all these for catching threading issues
388 			while( running || changed || !samplesStack.isEmpty())
389 			{
390 				if( changed )
391 				{
392 					fireTableDataChanged();
393 					changed = false;
394 				}
395 				
396 				if( !running && samplesStack.isEmpty() )
397 					break;
398 
399 				try
400 				{
401 					Thread.sleep( updateFrequency < 1 ? 1000 : updateFrequency );
402 				}
403 				catch (InterruptedException e)
404 				{
405 					SoapUI.logError( e );
406 				}
407 			}
408 		}
409 	}
410 	
411 	private void stop()
412 	{
413 		running = false;
414 	}
415 	
416 	/***
417 	 * Collect testresult samples
418 	 * 
419 	 * @author Ole.Matzura
420 	 */
421 	
422 	private class InternalTestRunListener extends LoadTestRunListenerAdapter
423 	{
424 		public void beforeLoadTest(LoadTestRunner loadTestRunner, LoadTestRunContext context)
425 		{
426 			running = true;
427 			new Thread( updater, loadTestRunner.getLoadTest().getName() + " LoadTestStatistics Updater" ).start();
428 			new Thread( LoadTestStatistics.this ).start();
429 			
430 			currentThreadCountStartTime = System.currentTimeMillis();
431 			totalAverageSum = 0;
432 		}
433 
434 		@Override
435 		public void afterTestStep( LoadTestRunner loadTestRunner, LoadTestRunContext context, TestRunner testRunner, TestRunContext runContext, TestStepResult testStepResult )
436 		{
437 			super.afterTestStep( loadTestRunner, context, testRunner, runContext, testStepResult );
438 		}
439 
440 		public void afterTestCase(LoadTestRunner loadTestRunner, LoadTestRunContext context, TestRunner testRunner, TestRunContext runContext)
441 		{
442 			List<TestStepResult> results = testRunner.getResults();
443 			TestCase testCase = testRunner.getTestCase();
444 			
445 			long[] samples = new long[testCase.getTestStepCount()];
446 			long[] sizes = new long[samples.length];
447 			long[] sampleCounts = new long[samples.length];
448 			
449 			for( int c = 0; c < results.size(); c++ )
450 			{
451 				TestStepResult testStepResult = results.get( c );
452 				if( testStepResult == null )
453 				{
454 					log.warn( "Result [" + c + "] is null in TestCase [" + testCase.getName() + "]" );
455 					continue;
456 				}
457 				
458 				int index = testCase.getIndexOfTestStep( testStepResult.getTestStep() );
459 				sampleCounts[index]++;
460 				
461 				samples[index] += testStepResult.getTimeTaken();
462 				sizes[index] += testStepResult.getSize();
463 			}
464 			
465 			pushSamples( samples, sizes, sampleCounts, testRunner.getStartTime(), testRunner.getTimeTaken() );
466 		}
467 
468 		@Override
469 		public void afterLoadTest( LoadTestRunner loadTestRunner, LoadTestRunContext context )
470 		{
471 			stop();
472 		}
473 	}
474 
475 	public int getStepCount()
476 	{
477 		return loadTest.getTestCase().getTestStepCount();
478 	}
479 
480 	public void reset()
481 	{
482 		init();
483 		fireTableDataChanged();
484 	}
485 
486 	public void release()
487 	{
488 		reset();
489 		
490 		loadTest.removeLoadTestRunListener( testRunListener );
491 		loadTest.getTestCase().getTestSuite().removeTestSuiteListener( testSuiteListener );
492 		
493 		for( TestStep testStep : loadTest.getTestCase().getTestStepList() )
494 		{
495 			testStep.removePropertyChangeListener( propertyChangeListener );
496 		}
497 	}
498 	
499 	private class InternalTestSuiteListener extends TestSuiteListenerAdapter
500 	{
501 		public void testStepAdded(TestStep testStep, int index)
502 		{
503 			if( testStep.getTestCase() == loadTest.getTestCase() )
504 			{
505 				init();
506 				testStep.addPropertyChangeListener( TestStep.NAME_PROPERTY, propertyChangeListener );
507 				fireTableDataChanged();
508 				
509 				history.reset();
510 			}
511 		}
512 
513 		public void testStepRemoved(TestStep testStep, int index)
514 		{
515 			if( testStep.getTestCase() == loadTest.getTestCase() )
516 			{
517 				init();
518 				testStep.removePropertyChangeListener( propertyChangeListener );
519 				fireTableDataChanged();
520 
521 				history.reset();
522 			}
523 		}
524 	}
525 
526 	private class InternalPropertyChangeListener implements PropertyChangeListener
527 	{
528 		public void propertyChange(PropertyChangeEvent evt)
529 		{
530 			if( evt.getSource() == loadTest && evt.getPropertyName().equals( WsdlLoadTest.THREADCOUNT_PROPERTY ))
531 			{
532 				if( loadTest.getResetStatisticsOnThreadCountChange() )
533 				{
534 					resetStatistics = true;
535 					currentThreadCountStartTime = System.currentTimeMillis();
536 				}
537 			}
538 			else if( evt.getPropertyName().equals( TestStep.NAME_PROPERTY ) ||
539 						evt.getPropertyName().equals( TestStep.DISABLED_PROPERTY ))
540 			{
541 				if( evt.getSource() instanceof TestStep )
542 					fireTableCellUpdated( loadTest.getTestCase().getIndexOfTestStep( (TestStep)evt.getSource() ), 1 );
543 			}
544 			else if( evt.getPropertyName().equals( WsdlLoadTest.HISTORYLIMIT_PROPERTY ))
545 			{
546 				if( loadTest.getHistoryLimit() == 0 )
547 					history.reset();
548 			}
549 		}}
550 	
551 	public TestStep getTestStepAtRow(int selectedRow)
552 	{
553 		if( selectedRow < getRowCount()-1 )
554 			return loadTest.getTestCase().getTestStepAt( selectedRow );
555 		else
556 			return null;
557 	}
558 
559 	public long getUpdateFrequency()
560 	{
561 		return updateFrequency;
562 	}
563 
564 	public void setUpdateFrequency(long updateFrequency)
565 	{
566 		this.updateFrequency = updateFrequency;
567 	}
568 	
569 	public void addError( int stepIndex )
570 	{
571 		if( stepIndex != -1 )
572 		{
573 			data[stepIndex][ERR_COLUMN]++;
574 		}
575 		
576 		data[data.length-1][ERR_COLUMN]++;
577 		changed = true;
578 	}
579 
580 	public long[][] getSnapshot()
581 	{
582 		return data.clone();
583 	}
584 	
585 	private final static Map<Integer,Statistic> statisticIndexMap = new HashMap<Integer,Statistic>();
586 
587 	private Updater updater = new Updater();
588 
589 	public enum Statistic 
590 	{ 
591 		MININMUM( MIN_COLUMN, "min", "the minimum measured teststep time" ), 
592 		MAXIMUM( MAX_COLUMN, "max", "the maximum measured testste time" ), 
593 		AVERAGE( AVG_COLUMN, "avg", "the average measured teststep time"), 
594 		LAST( LAST_COLUMN, "last", "the last measured teststep time" ), 
595 		COUNT( CNT_COLUMN, "cnt", "the number of teststep samples measured" ), 
596 		TPS( TPS_COLUMN, "tps", "the number of transactions per second for this teststep" ), 
597 		BYTES( BYTES_COLUMN, "bytes", "the total number of bytes returned by this teststep" ), 
598 		BPS( BPS_COLUMN, "bps", "the number of bytes per second returned by this teststep"), 
599 		ERRORS( ERR_COLUMN, "err", "the total number of assertion errors for this teststep" );
600 		
601 		private final String description;
602 		private final String name;
603 		private final int index;
604 
605 		Statistic( int index, String name, String description )
606 		{
607 			this.index = index;
608 			this.name = name;
609 			this.description = description;
610 			
611 			statisticIndexMap.put( index, this );
612 		}
613 
614 		public String getDescription()
615 		{
616 			return description;
617 		}
618 
619 		public int getIndex()
620 		{
621 			return index;
622 		}
623 
624 		public String getName()
625 		{
626 			return name;
627 		}
628 
629 		public static Statistic forIndex(int column)
630 		{
631 			return statisticIndexMap.get( column );
632 		}
633 	}
634 	
635 	/***
636 	 * Holds all sample values for a testcase run
637 	 * @author ole.matzura
638 	 */
639 	
640 	private static final class SamplesHolder
641 	{
642 		private final long[] samples;
643 		private final long[] sizes;
644 		private final long[] sampleCounts;
645 		
646 		private long startTime;
647 		private long timeTaken;
648 
649 		public SamplesHolder(long[] samples, long[] sizes, long[] sampleCounts, long startTime, long timeTaken )
650 		{
651 			this.samples = samples;
652 			this.sizes = sizes;
653 			this.startTime = startTime;
654 			this.timeTaken = timeTaken;
655 			this.sampleCounts = sampleCounts;
656 		}
657 	}
658 
659 	public synchronized void finish()
660 	{
661 		int sz = samplesStack.size();
662 		while( sz > 0 )
663 		{
664 			log.info( "Waiting for " + sz + " samples.." );
665 			try
666 			{
667 				Thread.sleep( 500 );
668 			}
669 			catch (InterruptedException e)
670 			{
671 				SoapUI.logError( e );
672 			}
673 			 sz = samplesStack.size();
674 		}
675 	}
676 }