| 1 | |
|
| 2 | |
|
| 3 | |
|
| 4 | |
|
| 5 | |
|
| 6 | |
|
| 7 | |
|
| 8 | |
|
| 9 | |
|
| 10 | |
|
| 11 | |
|
| 12 | |
|
| 13 | |
|
| 14 | |
package net.admin4j.monitor; |
| 15 | |
|
| 16 | |
import java.util.ArrayList; |
| 17 | |
import java.util.Date; |
| 18 | |
import java.util.HashMap; |
| 19 | |
import java.util.List; |
| 20 | |
import java.util.Map; |
| 21 | |
|
| 22 | |
import net.admin4j.timer.SummaryDataMeasure; |
| 23 | |
import net.admin4j.util.GuiUtils; |
| 24 | |
import net.admin4j.util.HostUtils; |
| 25 | |
import net.admin4j.util.NumberUtils; |
| 26 | |
import net.admin4j.util.notify.Notifier; |
| 27 | |
import net.admin4j.util.threaddumper.ThreadDumperFactory; |
| 28 | |
import net.admin4j.vo.MemoryUtilizationVO; |
| 29 | |
|
| 30 | |
|
| 31 | |
|
| 32 | |
|
| 33 | |
|
| 34 | |
|
| 35 | |
public class LowMemoryDetector extends Detector { |
| 36 | |
|
| 37 | |
public static final int DEFAULT_MEMORY_USAGE_THRESHOLD_PCT=90; |
| 38 | 15 | private long memoryThresholdPct = DEFAULT_MEMORY_USAGE_THRESHOLD_PCT; |
| 39 | |
|
| 40 | |
public static final long DEFAULT_SLEEP_INTERVAL=30000; |
| 41 | |
|
| 42 | |
public static final int DEFAULT_NBR_INTERVALS_BETWEEN_WARNINGS=30; |
| 43 | |
private int nbrIntervalsBetweenWarnings; |
| 44 | |
|
| 45 | |
public static final long DEFAULT_STARTUP_PERIOD_IN_MILLIS=60000; |
| 46 | 15 | private long startupPeriodInMillis = DEFAULT_STARTUP_PERIOD_IN_MILLIS; |
| 47 | |
private Long monitorStartTimeMillis; |
| 48 | |
|
| 49 | |
public static final long DEFAULT_LOW_WATERMARK_MONITOR_INTERVAL_IN_MILLIS=1800000; |
| 50 | 15 | private long lowWatermarkMonitorIntervalInMillis = DEFAULT_LOW_WATERMARK_MONITOR_INTERVAL_IN_MILLIS; |
| 51 | |
|
| 52 | |
public static final int DEFAULT_NBR_LOW_WATERMARK_INTERVALS=48; |
| 53 | 15 | private int nbrLowWatermarkIntervals = DEFAULT_NBR_LOW_WATERMARK_INTERVALS; |
| 54 | 15 | private SummaryDataMeasure[] memoryMeasurement = new SummaryDataMeasure[nbrLowWatermarkIntervals]; |
| 55 | 15 | private int nbrLowWatermarkMeasurementOffset = -1; |
| 56 | |
|
| 57 | |
|
| 58 | |
|
| 59 | 15 | private int nbrIntervalsSinceLastWarning = 0; |
| 60 | 15 | private boolean lowMemoryStateDetected = false; |
| 61 | 15 | private Long timeMemoryProblemBeganInMillis = null; |
| 62 | |
|
| 63 | |
public LowMemoryDetector(Notifier notifier) { |
| 64 | 15 | super(notifier); |
| 65 | 15 | } |
| 66 | |
|
| 67 | |
public LowMemoryDetector(Notifier notifier, long memoryThresholdPct, int nbrIntervalsBetweenWarnings, int nbrLowWatermarkIntervals, long lowWatermarkMonitorIntervalInMillis) { |
| 68 | 6 | this(notifier); |
| 69 | 6 | this.setMemoryThresholdPct(memoryThresholdPct); |
| 70 | 6 | this.setNbrIntervalsBetweenWarnings(nbrIntervalsBetweenWarnings); |
| 71 | 6 | this.setNbrLowWatermarkIntervals(nbrLowWatermarkIntervals); |
| 72 | 6 | this.setLowWatermarkMonitorIntervalInMillis(lowWatermarkMonitorIntervalInMillis); |
| 73 | 6 | } |
| 74 | |
|
| 75 | |
public void run() { |
| 76 | 0 | if (monitorStartTimeMillis == null) { |
| 77 | 0 | monitorStartTimeMillis = System.currentTimeMillis(); |
| 78 | |
} |
| 79 | |
|
| 80 | 0 | MemoryUtilizationVO memoryVO = this.findMemoryUtilization(); |
| 81 | 0 | this.watchLowWatermark(memoryVO); |
| 82 | 0 | this.checkMemory(memoryVO); |
| 83 | 0 | } |
| 84 | |
|
| 85 | |
protected void watchLowWatermark(MemoryUtilizationVO memoryVO) { |
| 86 | 90 | if (monitorStartTimeMillis == null) { |
| 87 | 9 | monitorStartTimeMillis = System.currentTimeMillis(); |
| 88 | |
} |
| 89 | |
|
| 90 | 90 | if (System.currentTimeMillis() < monitorStartTimeMillis + startupPeriodInMillis) { |
| 91 | 0 | return; |
| 92 | |
} |
| 93 | |
|
| 94 | 90 | if (nbrLowWatermarkMeasurementOffset < 0 && System.currentTimeMillis() >= monitorStartTimeMillis + startupPeriodInMillis) { |
| 95 | 9 | nbrLowWatermarkMeasurementOffset = 0; |
| 96 | 9 | memoryMeasurement[nbrLowWatermarkMeasurementOffset] = new SummaryDataMeasure(System.currentTimeMillis()); |
| 97 | |
} |
| 98 | 81 | else if (System.currentTimeMillis() > memoryMeasurement[nbrLowWatermarkMeasurementOffset].getFirstObservationTime() + lowWatermarkMonitorIntervalInMillis) { |
| 99 | 81 | nbrLowWatermarkMeasurementOffset++; |
| 100 | 81 | if (nbrLowWatermarkMeasurementOffset >= memoryMeasurement.length) { |
| 101 | 0 | nbrLowWatermarkMeasurementOffset = 0; |
| 102 | |
} |
| 103 | 81 | memoryMeasurement[nbrLowWatermarkMeasurementOffset] = new SummaryDataMeasure(System.currentTimeMillis()); |
| 104 | |
} |
| 105 | |
|
| 106 | 90 | memoryMeasurement[nbrLowWatermarkMeasurementOffset].addNumber(memoryVO.getMemoryInUseInBytes()); |
| 107 | 90 | } |
| 108 | |
|
| 109 | |
private List<SummaryDataMeasure> getCurrentLowWatermarkMeasurementList() { |
| 110 | 21 | List<SummaryDataMeasure> measureList = new ArrayList<SummaryDataMeasure>(); |
| 111 | |
|
| 112 | 210 | for (int i = nbrLowWatermarkMeasurementOffset - 1; i >= 0; i--) { |
| 113 | 189 | if (memoryMeasurement[i] != null) { |
| 114 | 189 | measureList.add(memoryMeasurement[i]); |
| 115 | |
} |
| 116 | |
} |
| 117 | 819 | for (int i = memoryMeasurement.length - 1; i > nbrLowWatermarkMeasurementOffset; i--) { |
| 118 | 798 | if (memoryMeasurement[i] != null) { |
| 119 | 0 | measureList.add(memoryMeasurement[i]); |
| 120 | |
} |
| 121 | |
} |
| 122 | |
|
| 123 | 21 | return measureList; |
| 124 | |
} |
| 125 | |
|
| 126 | |
private double getAverageLowWatermarkIncrease(List<SummaryDataMeasure> measureList) { |
| 127 | 21 | List<Number> numberList = new ArrayList<Number>(); |
| 128 | 21 | Number lastNumber = null; |
| 129 | 21 | for (SummaryDataMeasure measurement: measureList) { |
| 130 | 189 | if (lastNumber != null) { |
| 131 | 168 | numberList.add(measurement.getMinimum() - lastNumber.doubleValue()); |
| 132 | |
} |
| 133 | 189 | lastNumber = measurement.getMinimum(); |
| 134 | |
} |
| 135 | |
|
| 136 | 21 | return NumberUtils.average(numberList).doubleValue(); |
| 137 | |
} |
| 138 | |
|
| 139 | |
protected void checkMemory(MemoryUtilizationVO memoryVO) { |
| 140 | |
|
| 141 | 3 | if (memoryVO.getPercentMemoryUsed() > this.getMemoryThresholdPct()) { |
| 142 | 3 | if (this.lowMemoryStateDetected && this.nbrIntervalsSinceLastWarning >= this.getNbrIntervalsBetweenWarnings()) { |
| 143 | 0 | this.sendLowMemoryNotice(memoryVO); |
| 144 | 0 | this.nbrIntervalsSinceLastWarning = 0; |
| 145 | |
} |
| 146 | 3 | if (this.lowMemoryStateDetected) { |
| 147 | 0 | this.nbrIntervalsSinceLastWarning++; |
| 148 | |
} |
| 149 | |
else { |
| 150 | 3 | this.lowMemoryStateDetected = true; |
| 151 | 3 | this.nbrIntervalsSinceLastWarning = 0; |
| 152 | 3 | this.timeMemoryProblemBeganInMillis = System.currentTimeMillis(); |
| 153 | 3 | this.sendLowMemoryNotice(memoryVO); |
| 154 | |
} |
| 155 | |
} |
| 156 | |
else { |
| 157 | 0 | if (this.lowMemoryStateDetected) { |
| 158 | 0 | this.sendAllClearMemoryNotice(memoryVO); |
| 159 | 0 | this.lowMemoryStateDetected = false; |
| 160 | 0 | this.timeMemoryProblemBeganInMillis = null; |
| 161 | |
} |
| 162 | |
} |
| 163 | 3 | } |
| 164 | |
|
| 165 | |
protected void sendLowMemoryNotice(MemoryUtilizationVO memoryUtilization) { |
| 166 | 12 | List<SummaryDataMeasure> lowMemoryMeasurementList = this.getCurrentLowWatermarkMeasurementList(); |
| 167 | 12 | Double averageLowMemoryIncrease = this.getAverageLowWatermarkIncrease(lowMemoryMeasurementList); |
| 168 | |
|
| 169 | 12 | Map<String, Object> variableMap = new HashMap<String, Object>(); |
| 170 | 12 | variableMap.put("host", HostUtils.getHostName()); |
| 171 | 12 | variableMap.put("memory", memoryUtilization); |
| 172 | 12 | variableMap.put("currentDatetime", new Date()); |
| 173 | 12 | variableMap.put("averageLowMemoryIncrease", averageLowMemoryIncrease); |
| 174 | 12 | variableMap.put("lowWatermarkMonitorIntervalInMillis", lowWatermarkMonitorIntervalInMillis); |
| 175 | 12 | variableMap.put("lowMemoryMeasurementList", lowMemoryMeasurementList); |
| 176 | 12 | variableMap.put("GuiUtils", new GuiUtils()); |
| 177 | 12 | variableMap.put("threadInfoArray", ThreadDumperFactory.getThreadDumper().dumpAllThreads()); |
| 178 | |
|
| 179 | 12 | if (this.getNotifier().supportsHtml()) { |
| 180 | 3 | this.sendMessage(HostUtils.getHostName() + ": High Memory Usage - " + memoryUtilization.getPercentMemoryUsed() + "%", "lowMemoryNoticeHtml.ftl", variableMap); |
| 181 | |
} |
| 182 | 9 | else if (this.getNotifier().supportsSMS()) { |
| 183 | 3 | this.getNotifier().notify(HostUtils.getHostName() + |
| 184 | |
": High Memory Usage - " + memoryUtilization.getPercentMemoryUsed() + "%", |
| 185 | |
"Free Memory=" + memoryUtilization.getFreeAvailableMemoryInMb() + " Mb out of " + memoryUtilization.getMaxMemoryInBytesInMb() + " Mb."); |
| 186 | |
} |
| 187 | |
else { |
| 188 | 6 | this.sendMessage(HostUtils.getHostName() + ": High Memory Usage - " + memoryUtilization.getPercentMemoryUsed() + "%", "lowMemoryNoticeText.ftl", variableMap); |
| 189 | |
} |
| 190 | 12 | } |
| 191 | |
|
| 192 | |
protected void sendAllClearMemoryNotice(MemoryUtilizationVO memoryUtilization) { |
| 193 | 9 | List<SummaryDataMeasure> lowMemoryMeasurementList = this.getCurrentLowWatermarkMeasurementList(); |
| 194 | 9 | Double averageLowMemoryIncrease = this.getAverageLowWatermarkIncrease(lowMemoryMeasurementList); |
| 195 | |
|
| 196 | 9 | Map<String, Object> variableMap = new HashMap<String, Object>(); |
| 197 | 9 | variableMap.put("host", HostUtils.getHostName()); |
| 198 | 9 | variableMap.put("memory", memoryUtilization); |
| 199 | 9 | variableMap.put("endDate", new Date()); |
| 200 | 9 | if (this.timeMemoryProblemBeganInMillis != null) { |
| 201 | 0 | variableMap.put("startDate", new Date(this.timeMemoryProblemBeganInMillis)); |
| 202 | |
} |
| 203 | |
else { |
| 204 | 9 | logger.error("Low Memory Detector: Start Time is null. Current substituted but all-clear notice is incorrect."); |
| 205 | 9 | variableMap.put("startDate", new Date()); |
| 206 | |
} |
| 207 | |
|
| 208 | 9 | variableMap.put("averageLowMemoryIncrease", averageLowMemoryIncrease); |
| 209 | 9 | variableMap.put("lowMemoryMeasurementList", lowMemoryMeasurementList); |
| 210 | 9 | variableMap.put("lowWatermarkMonitorIntervalInMillis", lowWatermarkMonitorIntervalInMillis); |
| 211 | 9 | variableMap.put("GuiUtils", new GuiUtils()); |
| 212 | 9 | variableMap.put("threadInfoArray", ThreadDumperFactory.getThreadDumper().dumpAllThreads()); |
| 213 | |
|
| 214 | 9 | if (this.getNotifier().supportsHtml()) { |
| 215 | 3 | this.sendMessage(HostUtils.getHostName() + " - Memory Usage Normal ", "lowMemoryAllClearNoticeHtml.ftl", variableMap); |
| 216 | |
} |
| 217 | 6 | else if (this.getNotifier().supportsSMS()) { |
| 218 | 3 | this.getNotifier().notify(HostUtils.getHostName() + |
| 219 | |
" - Memory Usage Normal ", |
| 220 | |
"Free Memory=" + memoryUtilization.getFreeAvailableMemoryInMb() + |
| 221 | |
" Mb out of " + memoryUtilization.getMaxMemoryInBytesInMb() + " Mb."); |
| 222 | |
} |
| 223 | |
else { |
| 224 | 3 | this.sendMessage(HostUtils.getHostName() + " - Memory Usage Normal ", "lowMemoryAllClearNoticeText.ftl", variableMap); |
| 225 | |
} |
| 226 | 9 | } |
| 227 | |
|
| 228 | |
private MemoryUtilizationVO findMemoryUtilization() { |
| 229 | |
|
| 230 | 0 | long currentMemoryAllocated = Runtime.getRuntime().totalMemory(); |
| 231 | 0 | long freeMemory = Runtime.getRuntime().freeMemory(); |
| 232 | 0 | long maxMemory = Runtime.getRuntime().maxMemory(); |
| 233 | |
|
| 234 | 0 | return new MemoryUtilizationVO(currentMemoryAllocated, freeMemory, maxMemory); |
| 235 | |
} |
| 236 | |
|
| 237 | |
public long getMemoryThresholdPct() { |
| 238 | 3 | return memoryThresholdPct; |
| 239 | |
} |
| 240 | |
|
| 241 | |
protected void setMemoryThresholdPct(long usedMemoryThresholdPct) { |
| 242 | 15 | this.memoryThresholdPct = usedMemoryThresholdPct; |
| 243 | 15 | } |
| 244 | |
|
| 245 | |
public int getNbrIntervalsBetweenWarnings() { |
| 246 | 0 | return nbrIntervalsBetweenWarnings; |
| 247 | |
} |
| 248 | |
|
| 249 | |
protected void setNbrIntervalsBetweenWarnings(int nbrIntervalsBetweenWarnings) { |
| 250 | 15 | this.nbrIntervalsBetweenWarnings = nbrIntervalsBetweenWarnings; |
| 251 | 15 | } |
| 252 | |
|
| 253 | |
public int getNbrIntervalsSinceLastWarning() { |
| 254 | 0 | return nbrIntervalsSinceLastWarning; |
| 255 | |
} |
| 256 | |
|
| 257 | |
public boolean isLowMemoryStateDetected() { |
| 258 | 0 | return lowMemoryStateDetected; |
| 259 | |
} |
| 260 | |
|
| 261 | |
public long getStartupPeriodInMillis() { |
| 262 | 0 | return startupPeriodInMillis; |
| 263 | |
} |
| 264 | |
|
| 265 | |
protected void setStartupPeriodInMillis(long startupPeriodInMillis) { |
| 266 | 9 | this.startupPeriodInMillis = startupPeriodInMillis; |
| 267 | 9 | } |
| 268 | |
|
| 269 | |
public long getLowWatermarkMonitorIntervalInMillis() { |
| 270 | 0 | return lowWatermarkMonitorIntervalInMillis; |
| 271 | |
} |
| 272 | |
|
| 273 | |
protected void setLowWatermarkMonitorIntervalInMillis( |
| 274 | |
long lowWatermarkMonitorIntervalInMillis) { |
| 275 | 15 | this.lowWatermarkMonitorIntervalInMillis = lowWatermarkMonitorIntervalInMillis; |
| 276 | 15 | } |
| 277 | |
|
| 278 | |
public int getNbrLowWatermarkIntervals() { |
| 279 | 0 | return nbrLowWatermarkIntervals; |
| 280 | |
} |
| 281 | |
|
| 282 | |
protected void setNbrLowWatermarkIntervals(int nbrLowWatermarkIntervals) { |
| 283 | 6 | this.nbrLowWatermarkIntervals = nbrLowWatermarkIntervals; |
| 284 | 6 | } |
| 285 | |
|
| 286 | |
} |