Also dump system process threads halfway through the watchdog interval
This gives us a snapshot of what the system process was doing after 30 seconds
of apparent inactivity as well as after 1 minute, to help distinguishing actual
deadlocks from too-slow progress, livelock, etc.
Change-Id: I19758861d1b25f298e88788e8f1c7ec7bf828823
diff --git a/services/java/com/android/server/Watchdog.java b/services/java/com/android/server/Watchdog.java
index be1d1c4..5eaadbc 100644
--- a/services/java/com/android/server/Watchdog.java
+++ b/services/java/com/android/server/Watchdog.java
@@ -54,7 +54,8 @@
static final int MONITOR = 2718;
static final int GLOBAL_PSS = 2719;
- static final int TIME_TO_WAIT = DB ? 15*1000 : 60*1000;
+ static final int TIME_TO_RESTART = DB ? 15*1000 : 60*1000;
+ static final int TIME_TO_WAIT = TIME_TO_RESTART / 2;
static final int MEMCHECK_DEFAULT_INTERVAL = DB ? 30 : 30*60; // 30 minutes
static final int MEMCHECK_DEFAULT_LOG_REALTIME_INTERVAL = DB ? 60 : 2*60*60; // 2 hours
@@ -792,6 +793,7 @@
@Override
public void run() {
+ boolean waitedHalf = false;
while (true) {
mCompleted = false;
mHandler.sendEmptyMessage(MONITOR);
@@ -801,7 +803,7 @@
// NOTE: We use uptimeMillis() here because we do not want to increment the time we
// wait while asleep. If the device is asleep then the thing that we are waiting
- // to timeout on is asleep as well and won't have a chance to run. Causing a false
+ // to timeout on is asleep as well and won't have a chance to run, causing a false
// positive on when to kill things.
long start = SystemClock.uptimeMillis();
while (timeout > 0 && !mForceKillSystem) {
@@ -815,6 +817,17 @@
if (mCompleted && !mForceKillSystem) {
// The monitors have returned.
+ waitedHalf = false;
+ continue;
+ }
+
+ if (!waitedHalf) {
+ // We've waited half the deadlock-detection interval. Pull a stack
+ // trace and wait another half.
+ ArrayList pids = new ArrayList();
+ pids.add(Process.myPid());
+ File stack = ActivityManagerService.dumpStackTraces(true, pids);
+ waitedHalf = true;
continue;
}
}
@@ -829,7 +842,9 @@
ArrayList pids = new ArrayList();
pids.add(Process.myPid());
if (mPhonePid > 0) pids.add(mPhonePid);
- File stack = ActivityManagerService.dumpStackTraces(pids);
+ // Pass !waitedHalf so that just in case we somehow wind up here without having
+ // dumped the halfway stacks, we properly re-initialize the trace file.
+ File stack = ActivityManagerService.dumpStackTraces(!waitedHalf, pids);
// Give some extra time to make sure the stack traces get written.
// The system's been hanging for a minute, another second or two won't hurt much.
@@ -845,6 +860,8 @@
} else {
Slog.w(TAG, "Debugger connected: Watchdog is *not* killing the system process");
}
+
+ waitedHalf = false;
}
}
}
diff --git a/services/java/com/android/server/am/ActivityManagerService.java b/services/java/com/android/server/am/ActivityManagerService.java
index 61ab197..8857c5f 100644
--- a/services/java/com/android/server/am/ActivityManagerService.java
+++ b/services/java/com/android/server/am/ActivityManagerService.java
@@ -4779,10 +4779,13 @@
/**
* If a stack trace dump file is configured, dump process stack traces.
+ * @param clearTraces causes the dump file to be erased prior to the new
+ * traces being written, if true; when false, the new traces will be
+ * appended to any existing file content.
* @param pids of dalvik VM processes to dump stack traces for
* @return file containing stack traces, or null if no dump file is configured
*/
- public static File dumpStackTraces(ArrayList<Integer> pids) {
+ public static File dumpStackTraces(boolean clearTraces, ArrayList<Integer> pids) {
String tracesPath = SystemProperties.get("dalvik.vm.stack-trace-file", null);
if (tracesPath == null || tracesPath.length() == 0) {
return null;
@@ -4794,7 +4797,7 @@
if (!tracesDir.exists()) tracesFile.mkdirs();
FileUtils.setPermissions(tracesDir.getPath(), 0775, -1, -1); // drwxrwxr-x
- if (tracesFile.exists()) tracesFile.delete();
+ if (clearTraces && tracesFile.exists()) tracesFile.delete();
tracesFile.createNewFile();
FileUtils.setPermissions(tracesFile.getPath(), 0666, -1, -1); // -rw-rw-rw-
} catch (IOException e) {
@@ -4869,7 +4872,7 @@
}
}
- File tracesFile = dumpStackTraces(pids);
+ File tracesFile = dumpStackTraces(true, pids);
// Log the ANR to the main log.
StringBuilder info = mStringBuilder;