Audio from the Android Mobile

server/src/main/aidl/android/content/IOnPrimaryClipChangedListener.aidl

@@ -0,0 +1,24 @@
+/**
+ * Copyright (c) 2008, The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package android.content;
+
+/**
+ * {@hide}
+ */
+oneway interface IOnPrimaryClipChangedListener {
+ void dispatchPrimaryClipChanged();
+}

server/src/main/aidl/android/view/IDisplayFoldListener.aidl

@@ -0,0 +1,26 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package android.view;
+
+/**
+ * {@hide}
+ */
+oneway interface IDisplayFoldListener
+{
+ /** Called when the foldedness of a display changes */
+ void onDisplayFoldChanged(int displayId, boolean folded);
+}

server/src/main/java/org/cagnulein/android_remote/AsyncProcessor.java

@@ -0,0 +1,18 @@
+package org.cagnulein.android_remote;
+
+public interface AsyncProcessor {
+ interface TerminationListener {
+ /**
+ * Notify processor termination
+ *
+ * @param fatalError {@code true} if this must cause the termination of the whole scrcpy-server.
+ */
+ void onTerminated(boolean fatalError);
+ }
+
+ void start(TerminationListener listener);
+
+ void stop();
+
+ void join() throws InterruptedException;
+}

server/src/main/java/org/cagnulein/android_remote/AudioCapture.java

@@ -0,0 +1,179 @@
+package org.cagnulein.android_remote;
+
+import org.cagnulein.android_remote.wrappers.ServiceManager;
+
+import android.annotation.SuppressLint;
+import android.annotation.TargetApi;
+import android.content.ComponentName;
+import android.content.Intent;
+import android.media.AudioFormat;
+import android.media.AudioRecord;
+import android.media.AudioTimestamp;
+import android.media.MediaCodec;
+import android.os.Build;
+import android.os.SystemClock;
+
+import java.nio.ByteBuffer;
+
+public final class AudioCapture {
+
+ public static final int SAMPLE_RATE = 48000;
+ public static final int CHANNEL_CONFIG = AudioFormat.CHANNEL_IN_STEREO;
+ public static final int CHANNELS = 2;
+ public static final int CHANNEL_MASK = AudioFormat.CHANNEL_IN_LEFT | AudioFormat.CHANNEL_IN_RIGHT;
+ public static final int ENCODING = AudioFormat.ENCODING_PCM_16BIT;
+ public static final int BYTES_PER_SAMPLE = 2;
+
+ // Never read more than 1024 samples, even if the buffer is bigger (that would increase latency).
+ // A lower value is useless, since the system captures audio samples by blocks of 1024 (so for example if we read by blocks of 256 samples, we
+ // receive 4 successive blocks without waiting, then we wait for the 4 next ones).
+ public static final int MAX_READ_SIZE = 1024 * CHANNELS * BYTES_PER_SAMPLE;
+
+ private static final long ONE_SAMPLE_US = (1000000 + SAMPLE_RATE - 1) / SAMPLE_RATE; // 1 sample in microseconds (used for fixing PTS)
+
+ private final int audioSource;
+
+ private AudioRecord recorder;
+
+ private final AudioTimestamp timestamp = new AudioTimestamp();
+ private long previousRecorderTimestamp = -1;
+ private long previousPts = 0;
+ private long nextPts = 0;
+
+ public AudioCapture(AudioSource audioSource) {
+ this.audioSource = audioSource.value();
+ }
+
+ private static AudioFormat createAudioFormat() {
+ AudioFormat.Builder builder = new AudioFormat.Builder();
+ builder.setEncoding(ENCODING);
+ builder.setSampleRate(SAMPLE_RATE);
+ builder.setChannelMask(CHANNEL_CONFIG);
+ return builder.build();
+ }
+
+ @TargetApi(Build.VERSION_CODES.M)
+ @SuppressLint({"WrongConstant", "MissingPermission"})
+ private static AudioRecord createAudioRecord(int audioSource) {
+ AudioRecord.Builder builder = new AudioRecord.Builder();
+ if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
+ // On older APIs, Workarounds.fillAppInfo() must be called beforehand
+ builder.setContext(FakeContext.get());
+ }
+ builder.setAudioSource(audioSource);
+ builder.setAudioFormat(createAudioFormat());
+ int minBufferSize = AudioRecord.getMinBufferSize(SAMPLE_RATE, CHANNEL_CONFIG, ENCODING);
+ // This buffer size does not impact latency
+ builder.setBufferSizeInBytes(8 * minBufferSize);
+ return builder.build();
+ }
+
+ private static void startWorkaroundAndroid11() {
+ // Android 11 requires Apps to be at foreground to record audio.
+ // Normally, each App has its own user ID, so Android checks whether the requesting App has the user ID that's at the foreground.
+ // But scrcpy server is NOT an App, it's a Java application started from Android shell, so it has the same user ID (2000) with Android
+ // shell ("com.android.shell").
+ // If there is an Activity from Android shell running at foreground, then the permission system will believe scrcpy is also in the
+ // foreground.
+ Intent intent = new Intent(Intent.ACTION_MAIN);
+ intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK);
+ intent.addCategory(Intent.CATEGORY_LAUNCHER);
+ intent.setComponent(new ComponentName(FakeContext.PACKAGE_NAME, "com.android.shell.HeapDumpActivity"));
+ ServiceManager.getActivityManager().startActivity(intent);
+ }
+
+ private static void stopWorkaroundAndroid11() {
+ ServiceManager.getActivityManager().forceStopPackage(FakeContext.PACKAGE_NAME);
+ }
+
+ private void tryStartRecording(int attempts, int delayMs) throws AudioCaptureForegroundException {
+ while (attempts-- > 0) {
+ // Wait for activity to start
+ SystemClock.sleep(delayMs);
+ try {
+ startRecording();
+ return; // it worked
+ } catch (UnsupportedOperationException e) {
+ if (attempts == 0) {
+ Ln.e("Failed to start audio capture");
+ Ln.e("On Android 11, audio capture must be started in the foreground, make sure that the device is unlocked when starting "
+ + "scrcpy.");
+ throw new AudioCaptureForegroundException();
+ } else {
+ Ln.d("Failed to start audio capture, retrying...");
+ }
+ }
+ }
+ }
+
+ private void startRecording() {
+ try {
+ recorder = createAudioRecord(audioSource);
+ } catch (NullPointerException e) {
+ // Creating an AudioRecord using an AudioRecord.Builder does not work on Vivo phones:
+ // - <https://github.com/Genymobile/scrcpy/issues/3805>
+ // - <https://github.com/Genymobile/scrcpy/pull/3862>
+ recorder = Workarounds.createAudioRecord(audioSource, SAMPLE_RATE, CHANNEL_CONFIG, CHANNELS, CHANNEL_MASK, ENCODING);
+ }
+ recorder.startRecording();
+ }
+
+ public void start() throws AudioCaptureForegroundException {
+ if (Build.VERSION.SDK_INT == Build.VERSION_CODES.R) {
+ startWorkaroundAndroid11();
+ try {
+ tryStartRecording(5, 100);
+ } finally {
+ stopWorkaroundAndroid11();
+ }
+ } else {
+ startRecording();
+ }
+ }
+
+ public void stop() {
+ if (recorder != null) {
+ // Will call .stop() if necessary, without throwing an IllegalStateException
+ recorder.release();
+ }
+ }
+
+ @TargetApi(Build.VERSION_CODES.N)
+ public int read(ByteBuffer directBuffer, MediaCodec.BufferInfo outBufferInfo) {
+ int r = recorder.read(directBuffer, MAX_READ_SIZE);
+ if (r <= 0) {
+ return r;
+ }
+
+ long pts;
+
+ int ret = recorder.getTimestamp(timestamp, AudioTimestamp.TIMEBASE_MONOTONIC);
+ if (ret == AudioRecord.SUCCESS && timestamp.nanoTime != previousRecorderTimestamp) {
+ pts = timestamp.nanoTime / 1000;
+ previousRecorderTimestamp = timestamp.nanoTime;
+ } else {
+ if (nextPts == 0) {
+ Ln.w("Could not get initial audio timestamp");
+ nextPts = System.nanoTime() / 1000;
+ }
+ // compute from previous timestamp and packet size
+ pts = nextPts;
+ }
+
+ long durationUs = r * 1000000L / (CHANNELS * BYTES_PER_SAMPLE * SAMPLE_RATE);
+ nextPts = pts + durationUs;
+
+ if (previousPts != 0 && pts < previousPts + ONE_SAMPLE_US) {
+ // Audio PTS may come from two sources:
+ // - recorder.getTimestamp() if the call works;
+ // - an estimation from the previous PTS and the packet size as a fallback.
+ //
+ // Therefore, the property that PTS are monotonically increasing is no guaranteed in corner cases, so enforce it.
+ pts = previousPts + ONE_SAMPLE_US;
+ }
+ previousPts = pts;
+
+ outBufferInfo.set(0, r, pts, 0);
+ return r;
+ }
+}

server/src/main/java/org/cagnulein/android_remote/AudioCaptureForegroundException.java

@@ -0,0 +1,7 @@
+package org.cagnulein.android_remote;
+
+/**
+ * Exception thrown if audio capture failed on Android 11 specifically because the running App (shell) was not in foreground.
+ */
+public class AudioCaptureForegroundException extends Exception {
+}

server/src/main/java/org/cagnulein/android_remote/AudioCodec.java

@@ -0,0 +1,49 @@
+package org.cagnulein.android_remote;
+
+import android.media.MediaFormat;
+
+public enum AudioCodec implements Codec {
+ OPUS(0x6f_70_75_73, "opus", MediaFormat.MIMETYPE_AUDIO_OPUS),
+ AAC(0x00_61_61_63, "aac", MediaFormat.MIMETYPE_AUDIO_AAC),
+ FLAC(0x66_6c_61_63, "flac", MediaFormat.MIMETYPE_AUDIO_FLAC),
+ RAW(0x00_72_61_77, "raw", MediaFormat.MIMETYPE_AUDIO_RAW);
+
+ private final int id; // 4-byte ASCII representation of the name
+ private final String name;
+ private final String mimeType;
+
+ AudioCodec(int id, String name, String mimeType) {
+ this.id = id;
+ this.name = name;
+ this.mimeType = mimeType;
+ }
+
+ @Override
+ public Type getType() {
+ return Type.AUDIO;
+ }
+
+ @Override
+ public int getId() {
+ return id;
+ }
+
+ @Override
+ public String getName() {
+ return name;
+ }
+
+ @Override
+ public String getMimeType() {
+ return mimeType;
+ }
+
+ public static AudioCodec findByName(String name) {
+ for (AudioCodec codec : values()) {
+ if (codec.name.equals(name)) {
+ return codec;
+ }
+ }
+ return null;
+ }
+}