Multi-window assist callback
Introducing a new callback in VoiceInteractionSession to
provide assist data for additional activities in the
foreground in a multiwindow setup.
PIP, docked windows and free-form windows (top-most)
will be queried for assist data and passed through the
new API to the Voice Interaction service.
Bug: 27718385
Change-Id: Ib4427c304611b75c2078dcb54f1f7e47ae7d9cfa
diff --git a/api/current.txt b/api/current.txt
index 53738dc..ae35e19 100644
--- a/api/current.txt
+++ b/api/current.txt
@@ -34941,6 +34941,7 @@
method public void onDestroy();
method public boolean[] onGetSupportedCommands(java.lang.String[]);
method public void onHandleAssist(android.os.Bundle, android.app.assist.AssistStructure, android.app.assist.AssistContent);
+ method public void onHandleAssistSecondary(android.os.Bundle, android.app.assist.AssistStructure, android.app.assist.AssistContent, int, int);
method public void onHandleScreenshot(android.graphics.Bitmap);
method public void onHide();
method public boolean onKeyDown(int, android.view.KeyEvent);
diff --git a/api/system-current.txt b/api/system-current.txt
index ea83257..89ae0dd 100644
--- a/api/system-current.txt
+++ b/api/system-current.txt
@@ -37515,6 +37515,7 @@
method public void onDestroy();
method public boolean[] onGetSupportedCommands(java.lang.String[]);
method public void onHandleAssist(android.os.Bundle, android.app.assist.AssistStructure, android.app.assist.AssistContent);
+ method public void onHandleAssistSecondary(android.os.Bundle, android.app.assist.AssistStructure, android.app.assist.AssistContent, int, int);
method public void onHandleScreenshot(android.graphics.Bitmap);
method public void onHide();
method public boolean onKeyDown(int, android.view.KeyEvent);
diff --git a/api/test-current.txt b/api/test-current.txt
index caf3231..3acbc2e 100644
--- a/api/test-current.txt
+++ b/api/test-current.txt
@@ -35014,6 +35014,7 @@
method public void onDestroy();
method public boolean[] onGetSupportedCommands(java.lang.String[]);
method public void onHandleAssist(android.os.Bundle, android.app.assist.AssistStructure, android.app.assist.AssistContent);
+ method public void onHandleAssistSecondary(android.os.Bundle, android.app.assist.AssistStructure, android.app.assist.AssistContent, int, int);
method public void onHandleScreenshot(android.graphics.Bitmap);
method public void onHide();
method public boolean onKeyDown(int, android.view.KeyEvent);
diff --git a/core/java/android/app/ActivityManagerInternal.java b/core/java/android/app/ActivityManagerInternal.java
index 5116634..374c4f6 100644
--- a/core/java/android/app/ActivityManagerInternal.java
+++ b/core/java/android/app/ActivityManagerInternal.java
@@ -23,6 +23,8 @@
import com.android.internal.app.IVoiceInteractor;
+import java.util.List;
+
/**
* Activity manager local system service interface.
*
@@ -125,4 +127,10 @@
* Callback for window manager to let activity manager know that the app transition is finished.
*/
public abstract void notifyAppTransitionFinished();
+
+ /**
+ * Returns the top activity from each of the currently visible stacks. The first entry will be
+ * the focused activity.
+ */
+ public abstract List<IBinder> getTopVisibleActivities();
}
diff --git a/core/java/android/app/ActivityManagerNative.java b/core/java/android/app/ActivityManagerNative.java
index 65d48e6..7bf03de 100644
--- a/core/java/android/app/ActivityManagerNative.java
+++ b/core/java/android/app/ActivityManagerNative.java
@@ -2411,8 +2411,11 @@
data.enforceInterface(IActivityManager.descriptor);
int requestType = data.readInt();
IResultReceiver receiver = IResultReceiver.Stub.asInterface(data.readStrongBinder());
+ Bundle receiverExtras = data.readBundle();
IBinder activityToken = data.readStrongBinder();
- boolean res = requestAssistContextExtras(requestType, receiver, activityToken);
+ boolean focused = data.readInt() == 1;
+ boolean res = requestAssistContextExtras(requestType, receiver, receiverExtras,
+ activityToken, focused);
reply.writeNoException();
reply.writeInt(res ? 1 : 0);
return true;
@@ -6080,13 +6083,16 @@
}
public boolean requestAssistContextExtras(int requestType, IResultReceiver receiver,
- IBinder activityToken) throws RemoteException {
+ Bundle receiverExtras,
+ IBinder activityToken, boolean focused) throws RemoteException {
Parcel data = Parcel.obtain();
Parcel reply = Parcel.obtain();
data.writeInterfaceToken(IActivityManager.descriptor);
data.writeInt(requestType);
data.writeStrongBinder(receiver.asBinder());
+ data.writeBundle(receiverExtras);
data.writeStrongBinder(activityToken);
+ data.writeInt(focused ? 1 : 0);
mRemote.transact(REQUEST_ASSIST_CONTEXT_EXTRAS_TRANSACTION, data, reply, 0);
reply.readException();
boolean res = reply.readInt() != 0;
diff --git a/core/java/android/app/IActivityManager.java b/core/java/android/app/IActivityManager.java
index 8ee6fd0..6975116 100644
--- a/core/java/android/app/IActivityManager.java
+++ b/core/java/android/app/IActivityManager.java
@@ -523,7 +523,8 @@
public Bundle getAssistContextExtras(int requestType) throws RemoteException;
public boolean requestAssistContextExtras(int requestType, IResultReceiver receiver,
- IBinder activityToken) throws RemoteException;
+ Bundle receiverExtras,
+ IBinder activityToken, boolean focused) throws RemoteException;
public void reportAssistContextExtras(IBinder token, Bundle extras,
AssistStructure structure, AssistContent content, Uri referrer) throws RemoteException;
diff --git a/core/java/android/service/voice/IVoiceInteractionSession.aidl b/core/java/android/service/voice/IVoiceInteractionSession.aidl
index dbc28f7..78e6bc3 100644
--- a/core/java/android/service/voice/IVoiceInteractionSession.aidl
+++ b/core/java/android/service/voice/IVoiceInteractionSession.aidl
@@ -30,7 +30,8 @@
oneway interface IVoiceInteractionSession {
void show(in Bundle sessionArgs, int flags, IVoiceInteractionSessionShowCallback showCallback);
void hide();
- void handleAssist(in Bundle assistData, in AssistStructure structure, in AssistContent content);
+ void handleAssist(in Bundle assistData, in AssistStructure structure, in AssistContent content,
+ int index, int count);
void handleScreenshot(in Bitmap screenshot);
void taskStarted(in Intent intent, int taskId);
void taskFinished(in Intent intent, int taskId);
diff --git a/core/java/android/service/voice/VoiceInteractionSession.java b/core/java/android/service/voice/VoiceInteractionSession.java
index 6ff9fe7..e354ab3 100644
--- a/core/java/android/service/voice/VoiceInteractionSession.java
+++ b/core/java/android/service/voice/VoiceInteractionSession.java
@@ -77,7 +77,7 @@
*/
public class VoiceInteractionSession implements KeyEvent.Callback, ComponentCallbacks2 {
static final String TAG = "VoiceInteractionSession";
- static final boolean DEBUG = true;
+ static final boolean DEBUG = false;
/**
* Flag received in {@link #onShow}: originator requested that the session be started with
@@ -110,6 +110,16 @@
*/
public static final int SHOW_SOURCE_ACTIVITY = 1<<4;
+ // Keys for Bundle values
+ /** @hide */
+ public static final String KEY_DATA = "data";
+ /** @hide */
+ public static final String KEY_STRUCTURE = "structure";
+ /** @hide */
+ public static final String KEY_CONTENT = "content";
+ /** @hide */
+ public static final String KEY_RECEIVER_EXTRAS = "receiverExtras";
+
final Context mContext;
final HandlerCaller mHandlerCaller;
@@ -229,7 +239,7 @@
@Override
public void handleAssist(final Bundle data, final AssistStructure structure,
- final AssistContent content) {
+ final AssistContent content, final int index, final int count) {
// We want to pre-warm the AssistStructure before handing it off to the main
// thread. We also want to do this on a separate thread, so that if the app
// is for some reason slow (due to slow filling in of async children in the
@@ -247,8 +257,9 @@
failure = e;
}
}
- mHandlerCaller.sendMessage(mHandlerCaller.obtainMessageOOOO(MSG_HANDLE_ASSIST,
- data, failure == null ? structure : null, failure, content));
+ mHandlerCaller.sendMessage(mHandlerCaller.obtainMessageOOOOII(MSG_HANDLE_ASSIST,
+ data, failure == null ? structure : null, failure, content,
+ index, count));
}
};
retriever.start();
@@ -831,9 +842,16 @@
case MSG_HANDLE_ASSIST:
args = (SomeArgs)msg.obj;
if (DEBUG) Log.d(TAG, "onHandleAssist: data=" + args.arg1
- + " structure=" + args.arg2 + " content=" + args.arg3);
- doOnHandleAssist((Bundle) args.arg1, (AssistStructure) args.arg2,
- (Throwable) args.arg3, (AssistContent) args.arg4);
+ + " structure=" + args.arg2 + " content=" + args.arg3
+ + " activityIndex=" + args.argi5 + " activityCount=" + args.argi6);
+ if (args.argi5 == 0) {
+ doOnHandleAssist((Bundle) args.arg1, (AssistStructure) args.arg2,
+ (Throwable) args.arg3, (AssistContent) args.arg4);
+ } else {
+ doOnHandleAssistSecondary((Bundle) args.arg1, (AssistStructure) args.arg2,
+ (Throwable) args.arg3, (AssistContent) args.arg4,
+ args.argi5, args.argi6);
+ }
break;
case MSG_HANDLE_SCREENSHOT:
if (DEBUG) Log.d(TAG, "onHandleScreenshot: " + msg.obj);
@@ -1320,6 +1338,14 @@
onHandleAssist(data, structure, content);
}
+ void doOnHandleAssistSecondary(Bundle data, AssistStructure structure, Throwable failure,
+ AssistContent content, int index, int count) {
+ if (failure != null) {
+ onAssistStructureFailure(failure);
+ }
+ onHandleAssistSecondary(data, structure, content, index, count);
+ }
+
/**
* Called when there has been a failure transferring the {@link AssistStructure} to
* the assistant. This may happen, for example, if the data is too large and results
@@ -1356,6 +1382,45 @@
}
/**
+ * Called to receive data from other applications that the user was or is interacting with,
+ * that are currently on the screen in a multi-window display environment, not including the
+ * currently focused activity. This could be
+ * a free-form window, a picture-in-picture window, or another window in a split-screen display.
+ * <p>
+ * This method is very similar to
+ * {@link #onHandleAssist} except that it is called
+ * for additional non-focused activities along with an index and count that indicates
+ * which additional activity the data is for. {@code index} will be between 1 and
+ * {@code count}-1 and this method is called once for each additional window, in no particular
+ * order. The {@code count} indicates how many windows to expect assist data for, including the
+ * top focused activity, which continues to be returned via {@link #onHandleAssist}.
+ * <p>
+ * To be responsive to assist requests, process assist data as soon as it is received,
+ * without waiting for all queued activities to return assist data.
+ *
+ * @param data Arbitrary data supplied by the app through
+ * {@link android.app.Activity#onProvideAssistData Activity.onProvideAssistData}.
+ * May be null if assist data has been disabled by the user or device policy.
+ * @param structure If available, the structure definition of all windows currently
+ * displayed by the app. May be null if assist data has been disabled by the user
+ * or device policy; will be an empty stub if the application has disabled assist
+ * by marking its window as secure.
+ * @param content Additional content data supplied by the app through
+ * {@link android.app.Activity#onProvideAssistContent Activity.onProvideAssistContent}.
+ * May be null if assist data has been disabled by the user or device policy; will
+ * not be automatically filled in with data from the app if the app has marked its
+ * window as secure.
+ * @param index the index of the additional activity that this data
+ * is for.
+ * @param count the total number of additional activities for which the assist data is being
+ * returned, including the focused activity that is returned via
+ * {@link #onHandleAssist}.
+ */
+ public void onHandleAssistSecondary(@Nullable Bundle data, @Nullable AssistStructure structure,
+ @Nullable AssistContent content, int index, int count) {
+ }
+
+ /**
* Called to receive a screenshot of what the user was currently viewing when an assist
* session is started. May be null if screenshots are disabled by the user, policy,
* or application. If the original show request did not specify
diff --git a/core/java/com/android/internal/os/HandlerCaller.java b/core/java/com/android/internal/os/HandlerCaller.java
index 113768e..c26fc3a 100644
--- a/core/java/com/android/internal/os/HandlerCaller.java
+++ b/core/java/com/android/internal/os/HandlerCaller.java
@@ -209,6 +209,18 @@
return mH.obtainMessage(what, 0, 0, args);
}
+ public Message obtainMessageOOOOII(int what, Object arg1, Object arg2,
+ Object arg3, Object arg4, int arg5, int arg6) {
+ SomeArgs args = SomeArgs.obtain();
+ args.arg1 = arg1;
+ args.arg2 = arg2;
+ args.arg3 = arg3;
+ args.arg4 = arg4;
+ args.argi5 = arg5;
+ args.argi6 = arg6;
+ return mH.obtainMessage(what, 0, 0, args);
+ }
+
public Message obtainMessageIIII(int what, int arg1, int arg2,
int arg3, int arg4) {
SomeArgs args = SomeArgs.obtain();
@@ -218,7 +230,7 @@
args.argi4 = arg4;
return mH.obtainMessage(what, 0, 0, args);
}
-
+
public Message obtainMessageIIIIII(int what, int arg1, int arg2,
int arg3, int arg4, int arg5, int arg6) {
SomeArgs args = SomeArgs.obtain();
@@ -230,7 +242,7 @@
args.argi6 = arg6;
return mH.obtainMessage(what, 0, 0, args);
}
-
+
public Message obtainMessageIIIIO(int what, int arg1, int arg2,
int arg3, int arg4, Object arg5) {
SomeArgs args = SomeArgs.obtain();
diff --git a/services/core/java/com/android/server/am/ActivityManagerService.java b/services/core/java/com/android/server/am/ActivityManagerService.java
index 0f48d218..5b5e175 100644
--- a/services/core/java/com/android/server/am/ActivityManagerService.java
+++ b/services/core/java/com/android/server/am/ActivityManagerService.java
@@ -628,13 +628,16 @@
public Bundle result = null;
public AssistStructure structure = null;
public AssistContent content = null;
+ public Bundle receiverExtras;
+
public PendingAssistExtras(ActivityRecord _activity, Bundle _extras, Intent _intent,
- String _hint, IResultReceiver _receiver, int _userHandle) {
+ String _hint, IResultReceiver _receiver, Bundle _receiverExtras, int _userHandle) {
activity = _activity;
extras = _extras;
intent = _intent;
hint = _hint;
receiver = _receiver;
+ receiverExtras = _receiverExtras;
userHandle = _userHandle;
}
@Override
@@ -11804,7 +11807,7 @@
@Override
public Bundle getAssistContextExtras(int requestType) {
PendingAssistExtras pae = enqueueAssistContext(requestType, null, null, null,
- null, UserHandle.getCallingUserId(), null, PENDING_ASSIST_EXTRAS_TIMEOUT);
+ null, null, true, UserHandle.getCallingUserId(), null, PENDING_ASSIST_EXTRAS_TIMEOUT);
if (pae == null) {
return null;
}
@@ -11868,14 +11871,17 @@
@Override
public boolean requestAssistContextExtras(int requestType, IResultReceiver receiver,
- IBinder activityToken) {
- return enqueueAssistContext(requestType, null, null, receiver, activityToken,
- UserHandle.getCallingUserId(), null, PENDING_ASSIST_EXTRAS_LONG_TIMEOUT) != null;
+ Bundle receiverExtras,
+ IBinder activityToken, boolean focused) {
+ return enqueueAssistContext(requestType, null, null, receiver, receiverExtras,
+ activityToken, focused,
+ UserHandle.getCallingUserId(), null, PENDING_ASSIST_EXTRAS_LONG_TIMEOUT)
+ != null;
}
private PendingAssistExtras enqueueAssistContext(int requestType, Intent intent, String hint,
- IResultReceiver receiver, IBinder activityToken, int userHandle, Bundle args,
- long timeout) {
+ IResultReceiver receiver, Bundle receiverExtras, IBinder activityToken, boolean focused,
+ int userHandle, Bundle args, long timeout) {
enforceCallingPermission(android.Manifest.permission.GET_TOP_ACTIVITY_INFO,
"enqueueAssistContext()");
synchronized (this) {
@@ -11888,14 +11894,24 @@
Slog.w(TAG, "getAssistContextExtras failed: no process for " + activity);
return null;
}
- if (activityToken != null) {
- ActivityRecord caller = ActivityRecord.forTokenLocked(activityToken);
- if (activity != caller) {
- Slog.w(TAG, "enqueueAssistContext failed: caller " + caller
- + " is not current top " + activity);
+ if (focused) {
+ if (activityToken != null) {
+ ActivityRecord caller = ActivityRecord.forTokenLocked(activityToken);
+ if (activity != caller) {
+ Slog.w(TAG, "enqueueAssistContext failed: caller " + caller
+ + " is not current top " + activity);
+ return null;
+ }
+ }
+ } else {
+ activity = ActivityRecord.forTokenLocked(activityToken);
+ if (activity == null) {
+ Slog.w(TAG, "enqueueAssistContext failed: activity for token=" + activityToken
+ + " couldn't be found");
return null;
}
}
+
PendingAssistExtras pae;
Bundle extras = new Bundle();
if (args != null) {
@@ -11903,7 +11919,8 @@
}
extras.putString(Intent.EXTRA_ASSIST_PACKAGE, activity.packageName);
extras.putInt(Intent.EXTRA_ASSIST_UID, activity.app.uid);
- pae = new PendingAssistExtras(activity, extras, intent, hint, receiver, userHandle);
+ pae = new PendingAssistExtras(activity, extras, intent, hint, receiver, receiverExtras,
+ userHandle);
try {
activity.app.thread.requestAssistContextExtras(activity.appToken, pae,
requestType);
@@ -11973,9 +11990,11 @@
if ((sendReceiver=pae.receiver) != null) {
// Caller wants result sent back to them.
sendBundle = new Bundle();
- sendBundle.putBundle("data", pae.extras);
- sendBundle.putParcelable("structure", pae.structure);
- sendBundle.putParcelable("content", pae.content);
+ sendBundle.putBundle(VoiceInteractionSession.KEY_DATA, pae.extras);
+ sendBundle.putParcelable(VoiceInteractionSession.KEY_STRUCTURE, pae.structure);
+ sendBundle.putParcelable(VoiceInteractionSession.KEY_CONTENT, pae.content);
+ sendBundle.putBundle(VoiceInteractionSession.KEY_RECEIVER_EXTRAS,
+ pae.receiverExtras);
}
}
if (sendReceiver != null) {
@@ -12005,8 +12024,8 @@
public boolean launchAssistIntent(Intent intent, int requestType, String hint, int userHandle,
Bundle args) {
- return enqueueAssistContext(requestType, intent, hint, null, null, userHandle, args,
- PENDING_ASSIST_EXTRAS_TIMEOUT) != null;
+ return enqueueAssistContext(requestType, intent, hint, null, null, null, true,
+ userHandle, args, PENDING_ASSIST_EXTRAS_TIMEOUT) != null;
}
public void registerProcessObserver(IProcessObserver observer) {
@@ -20947,6 +20966,13 @@
mStackSupervisor.notifyAppTransitionDone();
}
}
+
+ @Override
+ public List<IBinder> getTopVisibleActivities() {
+ synchronized (ActivityManagerService.this) {
+ return mStackSupervisor.getTopVisibleActivities();
+ }
+ }
}
private final class SleepTokenImpl extends SleepToken {
diff --git a/services/core/java/com/android/server/am/ActivityStackSupervisor.java b/services/core/java/com/android/server/am/ActivityStackSupervisor.java
index d34e8fc..7e4ffae 100644
--- a/services/core/java/com/android/server/am/ActivityStackSupervisor.java
+++ b/services/core/java/com/android/server/am/ActivityStackSupervisor.java
@@ -98,6 +98,7 @@
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.Set;
@@ -4270,4 +4271,31 @@
}
return result;
}
+
+ /**
+ * @return a list of activities which are the top ones in each visible stack. The first
+ * entry will be the focused activity.
+ */
+ public List<IBinder> getTopVisibleActivities() {
+ final ActivityDisplay display = mActivityDisplays.get(Display.DEFAULT_DISPLAY);
+ if (display == null) {
+ return Collections.EMPTY_LIST;
+ }
+ ArrayList<IBinder> topActivityTokens = new ArrayList<>();
+ final ArrayList<ActivityStack> stacks = display.mStacks;
+ for (int i = stacks.size() - 1; i >= 0; i--) {
+ ActivityStack stack = stacks.get(i);
+ if (stack.getStackVisibilityLocked(null) == ActivityStack.STACK_VISIBLE) {
+ ActivityRecord top = stack.topActivity();
+ if (top != null) {
+ if (stack == mFocusedStack) {
+ topActivityTokens.add(0, top.appToken);
+ } else {
+ topActivityTokens.add(top.appToken);
+ }
+ }
+ }
+ }
+ return topActivityTokens;
+ }
}
diff --git a/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerServiceImpl.java b/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerServiceImpl.java
index 1544723..1e9db18 100644
--- a/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerServiceImpl.java
+++ b/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionManagerServiceImpl.java
@@ -17,6 +17,7 @@
package com.android.server.voiceinteraction;
import android.app.ActivityManager;
+import android.app.ActivityManagerInternal;
import android.app.ActivityManagerNative;
import android.app.IActivityManager;
import android.content.BroadcastReceiver;
@@ -42,9 +43,11 @@
import com.android.internal.app.IVoiceInteractionSessionShowCallback;
import com.android.internal.app.IVoiceInteractor;
+import com.android.server.LocalServices;
import java.io.FileDescriptor;
import java.io.PrintWriter;
+import java.util.List;
class VoiceInteractionManagerServiceImpl implements VoiceInteractionSessionConnection.Callback {
final static String TAG = "VoiceInteractionServiceManager";
@@ -148,8 +151,14 @@
mActiveSession = new VoiceInteractionSessionConnection(mLock, mSessionComponentName,
mUser, mContext, this, mInfo.getServiceInfo().applicationInfo.uid, mHandler);
}
+ List<IBinder> activityTokens = null;
+ if (activityToken == null) {
+ // Let's get top activities from all visible stacks
+ activityTokens = LocalServices.getService(ActivityManagerInternal.class)
+ .getTopVisibleActivities();
+ }
return mActiveSession.showLocked(args, flags, mDisabledShowContext, showCallback,
- activityToken);
+ activityToken, activityTokens);
}
public boolean hideSessionLocked() {
diff --git a/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionSessionConnection.java b/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionSessionConnection.java
index e04f312..0922a12 100644
--- a/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionSessionConnection.java
+++ b/services/voiceinteraction/java/com/android/server/voiceinteraction/VoiceInteractionSessionConnection.java
@@ -45,6 +45,7 @@
import android.util.Slog;
import android.view.IWindowManager;
import android.view.WindowManager;
+
import com.android.internal.app.IAssistScreenshotReceiver;
import com.android.internal.app.IVoiceInteractionSessionShowCallback;
import com.android.internal.app.IVoiceInteractor;
@@ -55,10 +56,15 @@
import java.io.PrintWriter;
import java.util.ArrayList;
+import java.util.List;
final class VoiceInteractionSessionConnection implements ServiceConnection {
+
final static String TAG = "VoiceInteractionServiceManager";
+ private static final String KEY_RECEIVER_EXTRA_COUNT = "count";
+ private static final String KEY_RECEIVER_EXTRA_INDEX = "index";
+
final IBinder mToken = new Binder();
final Object mLock;
final ComponentName mSessionComponentName;
@@ -82,11 +88,27 @@
IVoiceInteractionSession mSession;
IVoiceInteractor mInteractor;
boolean mHaveAssistData;
- Bundle mAssistData;
+ int mPendingAssistDataCount;
+ ArrayList<AssistDataForActivity> mAssistData = new ArrayList<>();
boolean mHaveScreenshot;
Bitmap mScreenshot;
ArrayList<IVoiceInteractionSessionShowCallback> mPendingShowCallbacks = new ArrayList<>();
+ static class AssistDataForActivity {
+ int activityIndex;
+ int activityCount;
+ Bundle data;
+
+ public AssistDataForActivity(Bundle data) {
+ this.data = data;
+ Bundle receiverExtras = data.getBundle(VoiceInteractionSession.KEY_RECEIVER_EXTRAS);
+ if (receiverExtras != null) {
+ activityIndex = receiverExtras.getInt(KEY_RECEIVER_EXTRA_INDEX);
+ activityCount = receiverExtras.getInt(KEY_RECEIVER_EXTRA_COUNT);
+ }
+ }
+ }
+
IVoiceInteractionSessionShowCallback mShowCallback =
new IVoiceInteractionSessionShowCallback.Stub() {
@Override
@@ -125,7 +147,7 @@
synchronized (mLock) {
if (mShown) {
mHaveAssistData = true;
- mAssistData = resultData;
+ mAssistData.add(new AssistDataForActivity(resultData));
deliverSessionDataLocked();
}
}
@@ -198,7 +220,8 @@
}
public boolean showLocked(Bundle args, int flags, int disabledContext,
- IVoiceInteractionSessionShowCallback showCallback, IBinder activityToken) {
+ IVoiceInteractionSessionShowCallback showCallback, IBinder activityToken,
+ List<IBinder> topActivities) {
if (mBound) {
if (!mFullyBound) {
mFullyBound = mContext.bindServiceAsUser(mBindIntent, mFullConnection,
@@ -220,30 +243,41 @@
mShowArgs = args;
mShowFlags = flags;
mHaveAssistData = false;
+ mPendingAssistDataCount = 0;
boolean needDisclosure = false;
if ((flags&VoiceInteractionSession.SHOW_WITH_ASSIST) != 0) {
if (mAppOps.noteOpNoThrow(AppOpsManager.OP_ASSIST_STRUCTURE, mCallingUid,
mSessionComponentName.getPackageName()) == AppOpsManager.MODE_ALLOWED
&& structureEnabled) {
- try {
- MetricsLogger.count(mContext, "assist_with_context", 1);
- if (mAm.requestAssistContextExtras(ActivityManager.ASSIST_CONTEXT_FULL,
- mAssistReceiver, activityToken)) {
- needDisclosure = true;
- } else {
- // Wasn't allowed... given that, let's not do the screenshot either.
- mHaveAssistData = true;
- mAssistData = null;
- screenshotEnabled = false;
+ mAssistData.clear();
+ final int count = activityToken != null ? 1 : topActivities.size();
+ for (int i = 0; i < count; i++) {
+ IBinder topActivity = count == 1 ? activityToken : topActivities.get(i);
+ try {
+ MetricsLogger.count(mContext, "assist_with_context", 1);
+ Bundle receiverExtras = new Bundle();
+ receiverExtras.putInt(KEY_RECEIVER_EXTRA_INDEX, i);
+ receiverExtras.putInt(KEY_RECEIVER_EXTRA_COUNT, count);
+ if (mAm.requestAssistContextExtras(ActivityManager.ASSIST_CONTEXT_FULL,
+ mAssistReceiver, receiverExtras, topActivity, i == 0)) {
+ needDisclosure = true;
+ mPendingAssistDataCount++;
+ } else if (i == 0) {
+ // Wasn't allowed... given that, let's not do the screenshot either.
+ mHaveAssistData = true;
+ mAssistData.clear();
+ screenshotEnabled = false;
+ break;
+ }
+ } catch (RemoteException e) {
}
- } catch (RemoteException e) {
}
} else {
mHaveAssistData = true;
- mAssistData = null;
+ mAssistData.clear();
}
} else {
- mAssistData = null;
+ mAssistData.clear();
}
mHaveScreenshot = false;
if ((flags&VoiceInteractionSession.SHOW_WITH_SCREENSHOT) != 0) {
@@ -335,41 +369,26 @@
return;
}
if (mHaveAssistData) {
- Bundle assistData;
- AssistStructure structure;
- AssistContent content;
- if (mAssistData != null) {
- assistData = mAssistData.getBundle("data");
- structure = mAssistData.getParcelable("structure");
- content = mAssistData.getParcelable("content");
- int uid = mAssistData.getInt(Intent.EXTRA_ASSIST_UID, -1);
- if (uid >= 0 && content != null) {
- Intent intent = content.getIntent();
- if (intent != null) {
- ClipData data = intent.getClipData();
- if (data != null && Intent.isAccessUriMode(intent.getFlags())) {
- grantClipDataPermissions(data, intent.getFlags(), uid,
- mCallingUid, mSessionComponentName.getPackageName());
- }
- }
- ClipData data = content.getClipData();
- if (data != null) {
- grantClipDataPermissions(data,
- Intent.FLAG_GRANT_READ_URI_PERMISSION,
- uid, mCallingUid, mSessionComponentName.getPackageName());
- }
+ AssistDataForActivity assistData;
+ while (!mAssistData.isEmpty()) {
+ if (mPendingAssistDataCount <= 0) {
+ Slog.e(TAG, "mPendingAssistDataCount is " + mPendingAssistDataCount);
}
- } else {
- assistData = null;
- structure = null;
- content = null;
+ mPendingAssistDataCount--;
+ assistData = mAssistData.remove(0);
+ if (assistData.data == null) {
+ try {
+ mSession.handleAssist(null, null, null, assistData.activityIndex,
+ assistData.activityCount);
+ } catch (RemoteException e) {
+ }
+ } else {
+ deliverSessionDataLocked(assistData);
+ }
}
- try {
- mSession.handleAssist(assistData, structure, content);
- } catch (RemoteException e) {
- }
- mAssistData = null;
- mHaveAssistData = false;
+ if (mPendingAssistDataCount <= 0) {
+ mHaveAssistData = false;
+ } // else, more to come
}
if (mHaveScreenshot) {
try {
@@ -381,6 +400,37 @@
}
}
+ private void deliverSessionDataLocked(AssistDataForActivity assistDataForActivity) {
+ Bundle assistData = assistDataForActivity.data.getBundle(
+ VoiceInteractionSession.KEY_DATA);
+ AssistStructure structure = assistDataForActivity.data.getParcelable(
+ VoiceInteractionSession.KEY_STRUCTURE);
+ AssistContent content = assistDataForActivity.data.getParcelable(
+ VoiceInteractionSession.KEY_CONTENT);
+ int uid = assistDataForActivity.data.getInt(Intent.EXTRA_ASSIST_UID, -1);
+ if (uid >= 0 && content != null) {
+ Intent intent = content.getIntent();
+ if (intent != null) {
+ ClipData data = intent.getClipData();
+ if (data != null && Intent.isAccessUriMode(intent.getFlags())) {
+ grantClipDataPermissions(data, intent.getFlags(), uid,
+ mCallingUid, mSessionComponentName.getPackageName());
+ }
+ }
+ ClipData data = content.getClipData();
+ if (data != null) {
+ grantClipDataPermissions(data,
+ Intent.FLAG_GRANT_READ_URI_PERMISSION,
+ uid, mCallingUid, mSessionComponentName.getPackageName());
+ }
+ }
+ try {
+ mSession.handleAssist(assistData, structure, content,
+ assistDataForActivity.activityIndex, assistDataForActivity.activityCount);
+ } catch (RemoteException e) {
+ }
+ }
+
public boolean hideLocked() {
if (mBound) {
if (mShown) {
@@ -388,7 +438,7 @@
mShowArgs = null;
mShowFlags = 0;
mHaveAssistData = false;
- mAssistData = null;
+ mAssistData.clear();
if (mSession != null) {
try {
mSession.hide();
diff --git a/tests/VoiceInteraction/src/com/android/test/voiceinteraction/MainInteractionSession.java b/tests/VoiceInteraction/src/com/android/test/voiceinteraction/MainInteractionSession.java
index 450334c..5767f11 100644
--- a/tests/VoiceInteraction/src/com/android/test/voiceinteraction/MainInteractionSession.java
+++ b/tests/VoiceInteraction/src/com/android/test/voiceinteraction/MainInteractionSession.java
@@ -194,6 +194,18 @@
}
@Override
+ public void onHandleAssistSecondary(final Bundle data, final AssistStructure structure,
+ final AssistContent content, int index, int count) {
+ Log.i(TAG, "Got secondary activity assist data " + index + " of " + count);
+ Log.i(TAG, "Showing assist structure after a few seconds...");
+ mContentView.postDelayed(new Runnable() {
+ public void run() {
+ onHandleAssist(data, structure, content);
+ }
+ }, 2000 * index);
+ }
+
+ @Override
public void onHandleScreenshot(Bitmap screenshot) {
if (screenshot != null) {
mScreenshot.setImageBitmap(screenshot);