1 | |
|
2 | |
|
3 | |
|
4 | |
|
5 | |
|
6 | |
|
7 | |
|
8 | |
|
9 | |
|
10 | |
|
11 | |
|
12 | |
|
13 | |
|
14 | |
package net.admin4j.monitor; |
15 | |
|
16 | |
import java.util.ArrayList; |
17 | |
import java.util.Date; |
18 | |
import java.util.HashMap; |
19 | |
import java.util.List; |
20 | |
import java.util.Map; |
21 | |
|
22 | |
import net.admin4j.timer.SummaryDataMeasure; |
23 | |
import net.admin4j.util.GuiUtils; |
24 | |
import net.admin4j.util.HostUtils; |
25 | |
import net.admin4j.util.NumberUtils; |
26 | |
import net.admin4j.util.notify.Notifier; |
27 | |
import net.admin4j.util.threaddumper.ThreadDumperFactory; |
28 | |
import net.admin4j.vo.MemoryUtilizationVO; |
29 | |
|
30 | |
|
31 | |
|
32 | |
|
33 | |
|
34 | |
|
35 | |
public class LowMemoryDetector extends Detector { |
36 | |
|
37 | |
public static final int DEFAULT_MEMORY_USAGE_THRESHOLD_PCT=90; |
38 | 15 | private long memoryThresholdPct = DEFAULT_MEMORY_USAGE_THRESHOLD_PCT; |
39 | |
|
40 | |
public static final long DEFAULT_SLEEP_INTERVAL=30000; |
41 | |
|
42 | |
public static final int DEFAULT_NBR_INTERVALS_BETWEEN_WARNINGS=30; |
43 | |
private int nbrIntervalsBetweenWarnings; |
44 | |
|
45 | |
public static final long DEFAULT_STARTUP_PERIOD_IN_MILLIS=60000; |
46 | 15 | private long startupPeriodInMillis = DEFAULT_STARTUP_PERIOD_IN_MILLIS; |
47 | |
private Long monitorStartTimeMillis; |
48 | |
|
49 | |
public static final long DEFAULT_LOW_WATERMARK_MONITOR_INTERVAL_IN_MILLIS=1800000; |
50 | 15 | private long lowWatermarkMonitorIntervalInMillis = DEFAULT_LOW_WATERMARK_MONITOR_INTERVAL_IN_MILLIS; |
51 | |
|
52 | |
public static final int DEFAULT_NBR_LOW_WATERMARK_INTERVALS=48; |
53 | 15 | private int nbrLowWatermarkIntervals = DEFAULT_NBR_LOW_WATERMARK_INTERVALS; |
54 | 15 | private SummaryDataMeasure[] memoryMeasurement = new SummaryDataMeasure[nbrLowWatermarkIntervals]; |
55 | 15 | private int nbrLowWatermarkMeasurementOffset = -1; |
56 | |
|
57 | |
|
58 | |
|
59 | 15 | private int nbrIntervalsSinceLastWarning = 0; |
60 | 15 | private boolean lowMemoryStateDetected = false; |
61 | 15 | private Long timeMemoryProblemBeganInMillis = null; |
62 | |
|
63 | |
public LowMemoryDetector(Notifier notifier) { |
64 | 15 | super(notifier); |
65 | 15 | } |
66 | |
|
67 | |
public LowMemoryDetector(Notifier notifier, long memoryThresholdPct, int nbrIntervalsBetweenWarnings, int nbrLowWatermarkIntervals, long lowWatermarkMonitorIntervalInMillis) { |
68 | 6 | this(notifier); |
69 | 6 | this.setMemoryThresholdPct(memoryThresholdPct); |
70 | 6 | this.setNbrIntervalsBetweenWarnings(nbrIntervalsBetweenWarnings); |
71 | 6 | this.setNbrLowWatermarkIntervals(nbrLowWatermarkIntervals); |
72 | 6 | this.setLowWatermarkMonitorIntervalInMillis(lowWatermarkMonitorIntervalInMillis); |
73 | 6 | } |
74 | |
|
75 | |
public void run() { |
76 | 0 | if (monitorStartTimeMillis == null) { |
77 | 0 | monitorStartTimeMillis = System.currentTimeMillis(); |
78 | |
} |
79 | |
|
80 | 0 | MemoryUtilizationVO memoryVO = this.findMemoryUtilization(); |
81 | 0 | this.watchLowWatermark(memoryVO); |
82 | 0 | this.checkMemory(memoryVO); |
83 | 0 | } |
84 | |
|
85 | |
protected void watchLowWatermark(MemoryUtilizationVO memoryVO) { |
86 | 90 | if (monitorStartTimeMillis == null) { |
87 | 9 | monitorStartTimeMillis = System.currentTimeMillis(); |
88 | |
} |
89 | |
|
90 | 90 | if (System.currentTimeMillis() < monitorStartTimeMillis + startupPeriodInMillis) { |
91 | 0 | return; |
92 | |
} |
93 | |
|
94 | 90 | if (nbrLowWatermarkMeasurementOffset < 0 && System.currentTimeMillis() >= monitorStartTimeMillis + startupPeriodInMillis) { |
95 | 9 | nbrLowWatermarkMeasurementOffset = 0; |
96 | 9 | memoryMeasurement[nbrLowWatermarkMeasurementOffset] = new SummaryDataMeasure(System.currentTimeMillis()); |
97 | |
} |
98 | 81 | else if (System.currentTimeMillis() > memoryMeasurement[nbrLowWatermarkMeasurementOffset].getFirstObservationTime() + lowWatermarkMonitorIntervalInMillis) { |
99 | 81 | nbrLowWatermarkMeasurementOffset++; |
100 | 81 | if (nbrLowWatermarkMeasurementOffset >= memoryMeasurement.length) { |
101 | 0 | nbrLowWatermarkMeasurementOffset = 0; |
102 | |
} |
103 | 81 | memoryMeasurement[nbrLowWatermarkMeasurementOffset] = new SummaryDataMeasure(System.currentTimeMillis()); |
104 | |
} |
105 | |
|
106 | 90 | memoryMeasurement[nbrLowWatermarkMeasurementOffset].addNumber(memoryVO.getMemoryInUseInBytes()); |
107 | 90 | } |
108 | |
|
109 | |
private List<SummaryDataMeasure> getCurrentLowWatermarkMeasurementList() { |
110 | 21 | List<SummaryDataMeasure> measureList = new ArrayList<SummaryDataMeasure>(); |
111 | |
|
112 | 210 | for (int i = nbrLowWatermarkMeasurementOffset - 1; i >= 0; i--) { |
113 | 189 | if (memoryMeasurement[i] != null) { |
114 | 189 | measureList.add(memoryMeasurement[i]); |
115 | |
} |
116 | |
} |
117 | 819 | for (int i = memoryMeasurement.length - 1; i > nbrLowWatermarkMeasurementOffset; i--) { |
118 | 798 | if (memoryMeasurement[i] != null) { |
119 | 0 | measureList.add(memoryMeasurement[i]); |
120 | |
} |
121 | |
} |
122 | |
|
123 | 21 | return measureList; |
124 | |
} |
125 | |
|
126 | |
private double getAverageLowWatermarkIncrease(List<SummaryDataMeasure> measureList) { |
127 | 21 | List<Number> numberList = new ArrayList<Number>(); |
128 | 21 | Number lastNumber = null; |
129 | 21 | for (SummaryDataMeasure measurement: measureList) { |
130 | 189 | if (lastNumber != null) { |
131 | 168 | numberList.add(measurement.getMinimum() - lastNumber.doubleValue()); |
132 | |
} |
133 | 189 | lastNumber = measurement.getMinimum(); |
134 | |
} |
135 | |
|
136 | 21 | return NumberUtils.average(numberList).doubleValue(); |
137 | |
} |
138 | |
|
139 | |
protected void checkMemory(MemoryUtilizationVO memoryVO) { |
140 | |
|
141 | 3 | if (memoryVO.getPercentMemoryUsed() > this.getMemoryThresholdPct()) { |
142 | 3 | if (this.lowMemoryStateDetected && this.nbrIntervalsSinceLastWarning >= this.getNbrIntervalsBetweenWarnings()) { |
143 | 0 | this.sendLowMemoryNotice(memoryVO); |
144 | 0 | this.nbrIntervalsSinceLastWarning = 0; |
145 | |
} |
146 | 3 | if (this.lowMemoryStateDetected) { |
147 | 0 | this.nbrIntervalsSinceLastWarning++; |
148 | |
} |
149 | |
else { |
150 | 3 | this.lowMemoryStateDetected = true; |
151 | 3 | this.nbrIntervalsSinceLastWarning = 0; |
152 | 3 | this.timeMemoryProblemBeganInMillis = System.currentTimeMillis(); |
153 | 3 | this.sendLowMemoryNotice(memoryVO); |
154 | |
} |
155 | |
} |
156 | |
else { |
157 | 0 | if (this.lowMemoryStateDetected) { |
158 | 0 | this.sendAllClearMemoryNotice(memoryVO); |
159 | 0 | this.lowMemoryStateDetected = false; |
160 | 0 | this.timeMemoryProblemBeganInMillis = null; |
161 | |
} |
162 | |
} |
163 | 3 | } |
164 | |
|
165 | |
protected void sendLowMemoryNotice(MemoryUtilizationVO memoryUtilization) { |
166 | 12 | List<SummaryDataMeasure> lowMemoryMeasurementList = this.getCurrentLowWatermarkMeasurementList(); |
167 | 12 | Double averageLowMemoryIncrease = this.getAverageLowWatermarkIncrease(lowMemoryMeasurementList); |
168 | |
|
169 | 12 | Map<String, Object> variableMap = new HashMap<String, Object>(); |
170 | 12 | variableMap.put("host", HostUtils.getHostName()); |
171 | 12 | variableMap.put("memory", memoryUtilization); |
172 | 12 | variableMap.put("currentDatetime", new Date()); |
173 | 12 | variableMap.put("averageLowMemoryIncrease", averageLowMemoryIncrease); |
174 | 12 | variableMap.put("lowWatermarkMonitorIntervalInMillis", lowWatermarkMonitorIntervalInMillis); |
175 | 12 | variableMap.put("lowMemoryMeasurementList", lowMemoryMeasurementList); |
176 | 12 | variableMap.put("GuiUtils", new GuiUtils()); |
177 | 12 | variableMap.put("threadInfoArray", ThreadDumperFactory.getThreadDumper().dumpAllThreads()); |
178 | |
|
179 | 12 | if (this.getNotifier().supportsHtml()) { |
180 | 3 | this.sendMessage(HostUtils.getHostName() + ": High Memory Usage - " + memoryUtilization.getPercentMemoryUsed() + "%", "lowMemoryNoticeHtml.ftl", variableMap); |
181 | |
} |
182 | 9 | else if (this.getNotifier().supportsSMS()) { |
183 | 3 | this.getNotifier().notify(HostUtils.getHostName() + |
184 | |
": High Memory Usage - " + memoryUtilization.getPercentMemoryUsed() + "%", |
185 | |
"Free Memory=" + memoryUtilization.getFreeAvailableMemoryInMb() + " Mb out of " + memoryUtilization.getMaxMemoryInBytesInMb() + " Mb."); |
186 | |
} |
187 | |
else { |
188 | 6 | this.sendMessage(HostUtils.getHostName() + ": High Memory Usage - " + memoryUtilization.getPercentMemoryUsed() + "%", "lowMemoryNoticeText.ftl", variableMap); |
189 | |
} |
190 | 12 | } |
191 | |
|
192 | |
protected void sendAllClearMemoryNotice(MemoryUtilizationVO memoryUtilization) { |
193 | 9 | List<SummaryDataMeasure> lowMemoryMeasurementList = this.getCurrentLowWatermarkMeasurementList(); |
194 | 9 | Double averageLowMemoryIncrease = this.getAverageLowWatermarkIncrease(lowMemoryMeasurementList); |
195 | |
|
196 | 9 | Map<String, Object> variableMap = new HashMap<String, Object>(); |
197 | 9 | variableMap.put("host", HostUtils.getHostName()); |
198 | 9 | variableMap.put("memory", memoryUtilization); |
199 | 9 | variableMap.put("endDate", new Date()); |
200 | 9 | if (this.timeMemoryProblemBeganInMillis != null) { |
201 | 0 | variableMap.put("startDate", new Date(this.timeMemoryProblemBeganInMillis)); |
202 | |
} |
203 | |
else { |
204 | 9 | logger.error("Low Memory Detector: Start Time is null. Current substituted but all-clear notice is incorrect."); |
205 | 9 | variableMap.put("startDate", new Date()); |
206 | |
} |
207 | |
|
208 | 9 | variableMap.put("averageLowMemoryIncrease", averageLowMemoryIncrease); |
209 | 9 | variableMap.put("lowMemoryMeasurementList", lowMemoryMeasurementList); |
210 | 9 | variableMap.put("lowWatermarkMonitorIntervalInMillis", lowWatermarkMonitorIntervalInMillis); |
211 | 9 | variableMap.put("GuiUtils", new GuiUtils()); |
212 | 9 | variableMap.put("threadInfoArray", ThreadDumperFactory.getThreadDumper().dumpAllThreads()); |
213 | |
|
214 | 9 | if (this.getNotifier().supportsHtml()) { |
215 | 3 | this.sendMessage(HostUtils.getHostName() + " - Memory Usage Normal ", "lowMemoryAllClearNoticeHtml.ftl", variableMap); |
216 | |
} |
217 | 6 | else if (this.getNotifier().supportsSMS()) { |
218 | 3 | this.getNotifier().notify(HostUtils.getHostName() + |
219 | |
" - Memory Usage Normal ", |
220 | |
"Free Memory=" + memoryUtilization.getFreeAvailableMemoryInMb() + |
221 | |
" Mb out of " + memoryUtilization.getMaxMemoryInBytesInMb() + " Mb."); |
222 | |
} |
223 | |
else { |
224 | 3 | this.sendMessage(HostUtils.getHostName() + " - Memory Usage Normal ", "lowMemoryAllClearNoticeText.ftl", variableMap); |
225 | |
} |
226 | 9 | } |
227 | |
|
228 | |
private MemoryUtilizationVO findMemoryUtilization() { |
229 | |
|
230 | 0 | long currentMemoryAllocated = Runtime.getRuntime().totalMemory(); |
231 | 0 | long freeMemory = Runtime.getRuntime().freeMemory(); |
232 | 0 | long maxMemory = Runtime.getRuntime().maxMemory(); |
233 | |
|
234 | 0 | return new MemoryUtilizationVO(currentMemoryAllocated, freeMemory, maxMemory); |
235 | |
} |
236 | |
|
237 | |
public long getMemoryThresholdPct() { |
238 | 3 | return memoryThresholdPct; |
239 | |
} |
240 | |
|
241 | |
protected void setMemoryThresholdPct(long usedMemoryThresholdPct) { |
242 | 15 | this.memoryThresholdPct = usedMemoryThresholdPct; |
243 | 15 | } |
244 | |
|
245 | |
public int getNbrIntervalsBetweenWarnings() { |
246 | 0 | return nbrIntervalsBetweenWarnings; |
247 | |
} |
248 | |
|
249 | |
protected void setNbrIntervalsBetweenWarnings(int nbrIntervalsBetweenWarnings) { |
250 | 15 | this.nbrIntervalsBetweenWarnings = nbrIntervalsBetweenWarnings; |
251 | 15 | } |
252 | |
|
253 | |
public int getNbrIntervalsSinceLastWarning() { |
254 | 0 | return nbrIntervalsSinceLastWarning; |
255 | |
} |
256 | |
|
257 | |
public boolean isLowMemoryStateDetected() { |
258 | 0 | return lowMemoryStateDetected; |
259 | |
} |
260 | |
|
261 | |
public long getStartupPeriodInMillis() { |
262 | 0 | return startupPeriodInMillis; |
263 | |
} |
264 | |
|
265 | |
protected void setStartupPeriodInMillis(long startupPeriodInMillis) { |
266 | 9 | this.startupPeriodInMillis = startupPeriodInMillis; |
267 | 9 | } |
268 | |
|
269 | |
public long getLowWatermarkMonitorIntervalInMillis() { |
270 | 0 | return lowWatermarkMonitorIntervalInMillis; |
271 | |
} |
272 | |
|
273 | |
protected void setLowWatermarkMonitorIntervalInMillis( |
274 | |
long lowWatermarkMonitorIntervalInMillis) { |
275 | 15 | this.lowWatermarkMonitorIntervalInMillis = lowWatermarkMonitorIntervalInMillis; |
276 | 15 | } |
277 | |
|
278 | |
public int getNbrLowWatermarkIntervals() { |
279 | 0 | return nbrLowWatermarkIntervals; |
280 | |
} |
281 | |
|
282 | |
protected void setNbrLowWatermarkIntervals(int nbrLowWatermarkIntervals) { |
283 | 6 | this.nbrLowWatermarkIntervals = nbrLowWatermarkIntervals; |
284 | 6 | } |
285 | |
|
286 | |
} |