encapp: x264 encoder current state

Signed-off-by: JohanBlome <[email protected]>

app/src/main/java/com/facebook/encapp/SwLibEncoder.java → app/src/main/java/com/facebook/encapp/CustomEncoder.java

@@ -3,29 +3,31 @@
 import static com.facebook.encapp.utils.TestDefinitionHelper.magnitudeToInt;
 
 import android.media.MediaCodec;
-import android.media.MediaCodecInfo;
 import android.media.MediaFormat;
-import android.media.MediaMuxer;
-import android.os.Environment;
+import android.os.Bundle;
 import android.util.Log;
 import android.util.Size;
 
 import androidx.annotation.NonNull;
 
 import com.facebook.encapp.proto.DataValueType;
 import com.facebook.encapp.proto.Parameter;
+import com.facebook.encapp.proto.PixFmt;
+import com.facebook.encapp.proto.Runtime;
 import com.facebook.encapp.proto.Test;
 import com.facebook.encapp.utils.FileReader;
+import com.facebook.encapp.utils.FrameInfo;
+import com.facebook.encapp.utils.MediaCodecInfoHelper;
 import com.facebook.encapp.utils.SizeUtils;
 import com.facebook.encapp.utils.Statistics;
+import com.facebook.encapp.utils.StringParameter;
 import com.facebook.encapp.utils.TestDefinitionHelper;
 
-import java.io.File;
 import java.io.FileInputStream;
-import java.io.FileOutputStream;
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.nio.channels.FileChannel;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Vector;
 
@@ -34,7 +36,7 @@
  * Created by jobl on 2018-02-27.
  */
 
-class SwLibEncoder extends Encoder {
+class CustomEncoder extends Encoder {
     protected static final String TAG = "encapp.buffer_x264_encoder";
 
     static{
@@ -46,14 +48,18 @@ class SwLibEncoder extends Encoder {
         }
     }
 
-    public static native int initEncoder(Parameter[] parameters, int width, int height, int colorSpace, int bitDepth);
+    public static native int initEncoder(Parameter[] parameters, int width, int height, int colorFormat, int bitDepth);
     public static native byte[] getHeader();
     // TODO: can the size, color and bitdepth change runtime?
-    public static native int encode(byte[] input, byte[] output, int width, int height, int colorSpace, int bitDepth);
+    public static native int encode(byte[] input, byte[] output, FrameInfo info);
+
+    public static native StringParameter[] getAllEncoderSettings();
+
+    public static native void updateSettings(Parameter[] parameters);
     public static native void close();
 
 
-    public SwLibEncoder(Test test) {
+    public CustomEncoder(Test test) {
         super(test);
         mStats = new Statistics("raw encoder", mTest);
     }
@@ -128,6 +134,38 @@ public boolean checkIfKeyFrame(byte[] bitstream) {
         return false;
     }
 
+
+    public void setRuntimeParameters(int frame) {
+        // go through all runtime settings and see which are due
+        if (mRuntimeParams == null) return;
+        Vector<Parameter> params = new Vector();
+
+        for (Runtime.VideoBitrateParameter bitrate : mRuntimeParams.getVideoBitrateList()) {
+            if (bitrate.getFramenum() == frame) {
+                params.add(Parameter.newBuilder().setKey("bitrate").setValue(String.valueOf(bitrate)).setType(DataValueType.stringType).build());
+                break;
+            }
+        }
+
+        for (Long sync : mRuntimeParams.getRequestSyncList()) {
+            if (sync == frame) {
+                params.add(Parameter.newBuilder().setKey("request-sync").setValue(String.valueOf("")).setType(DataValueType.stringType).build());
+                break;
+            }
+        }
+
+        for (Parameter param : mRuntimeParams.getParameterList()) {
+            if (param.getFramenum() == frame) {
+                Log.d(TAG, "Set runtime parameter @ " + frame + " key: " + param.getKey() + ", " + param.getType() + ", " + param.getValue());
+                // Easy everything (for now) is str
+            }
+        }
+
+        Parameter[] param_buffer = new Parameter[params.size()];
+        params.toArray(param_buffer);
+        updateSettings(param_buffer);
+    }
+
     public String start() {
         Log.d(TAG, "** Raw buffer encoding - " + mTest.getCommon().getDescription() + " **");
         try {
@@ -142,41 +180,39 @@ public String start() {
         if (mTest.getInput().hasRealtime())
             mRealtime = mTest.getInput().getRealtime();
 
-        boolean useImage = false;
-
         mFrameRate = mTest.getConfigure().getFramerate();
         mWriteFile = !mTest.getConfigure().hasEncode() || mTest.getConfigure().getEncode();
         mSkipped = 0;
         mFramesAdded = 0;
         Size sourceResolution = SizeUtils.parseXString(mTest.getInput().getResolution());
         mRefFramesizeInBytes = (int) (sourceResolution.getWidth() * sourceResolution.getHeight() * 1.5);
         mYuvReader = new FileReader();
+        //TODO: fix looping
         int playoutframes = mTest.getInput().getPlayoutFrames();
 
-        String preset = "fast";//mTest.getEncoderX264().getPreset();
         int width = sourceResolution.getWidth();
         int height = sourceResolution.getHeight();
-        int colorSpace = 2;//mTest.getEncoderX264().getColorSpace();
-        int bitdepth = 8;//mTest.getEncoderX264().getBitdepth();
+        int pixelformat = mTest.getInput().getPixFmt().getNumber();
+        int bitdepth = (pixelformat == PixFmt.p010le.getNumber())? 10: 8;
         int bitrate = magnitudeToInt(mTest.getConfigure().getBitrate());
-        int bitrateMode = mTest.getConfigure().getBitrateMode().getNumber();
+        int bitratemode = mTest.getConfigure().getBitrateMode().getNumber();
+        int iframeinterval =  mTest.getConfigure().getIFrameInterval();
 
         float crf = 23.0f;
-
-
+        Log.d(TAG, width + "x" + height + ",  pixelformat: " + pixelformat + ", bitdepth:" + bitdepth + ", bitrate_mode:" + bitratemode + ", bitrate: " + bitrate + ", iframeinterval: "+ iframeinterval);
         // Create params for this
-        // TODO: Translate from mediaformat into this (for common settings).
-        Vector<Parameter> params = new Vector<>();
-        params.add(Parameter.newBuilder().setKey("preset").setValue("fast").setType(DataValueType.stringType).build());
-        params.add(Parameter.newBuilder().setKey("tune").setValue("psnr").setType(DataValueType.stringType).build());
-        params.add(Parameter.newBuilder().setKey("i_threads").setValue("1").setType(DataValueType.stringType).build());
-        params.add(Parameter.newBuilder().setKey("i_width").setValue(String.valueOf(width)).setType(DataValueType.stringType).build());
-        params.add(Parameter.newBuilder().setKey("i_height").setValue(String.valueOf(height)).setType(DataValueType.stringType).build());
-        params.add(Parameter.newBuilder().setKey("i_csp").setValue(String.valueOf(colorSpace)).setType(DataValueType.stringType).build());
-        params.add(Parameter.newBuilder().setKey("i_bitdepth").setValue(String.valueOf(bitdepth)).setType(DataValueType.stringType).build());
-
-        params.add(Parameter.newBuilder().setKey("bitrate").setValue(String.valueOf(bitrate)).setType(DataValueType.stringType).build());
 
+
+        // Caching vital values and see if they can be set runtime.
+        Vector<Parameter> params = new Vector(mTest.getConfigure().getParameterList());
+        // This one needs to be set as a native param.
+        try {
+            params.add(Parameter.newBuilder().setKey("bitrate").setValue(String.valueOf(bitrate)).setType(DataValueType.stringType).build());
+            params.add(Parameter.newBuilder().setKey("bitrate_mode").setValue(String.valueOf(bitratemode)).setType(DataValueType.stringType).build());
+            params.add(Parameter.newBuilder().setKey("i_frame_interval").setValue(String.valueOf(iframeinterval)).setType(DataValueType.stringType).build());
+        } catch (Exception ex) {
+            Log.d(TAG, "Exception: " + ex);
+        }
         if (!mYuvReader.openFile(checkFilePath(mTest.getInput().getFilepath()), mTest.getInput().getPixFmt())) {
             return "Could not open file";
         }
@@ -219,13 +255,24 @@ public String start() {
 
             Parameter[] param_buffer = new Parameter[params.size()];
             params.toArray(param_buffer);
-            //TODO: We are setting width, height etc twice.
-            int status = initEncoder(param_buffer, width, height, colorSpace, bitdepth);
+            int status = initEncoder(param_buffer, width, height, pixelformat, bitdepth);
+            StringParameter[] settings_ = getAllEncoderSettings();
+            //add generic parameters to mTest, this way it can bre exactly reproduced (?) - in the case x264: no
+            ArrayList<Parameter> param_list = new ArrayList<>();
+            for (StringParameter par: settings_) {
+                param_list.add(par.getParameter());
+            }
+
+            // TODO: where to save this information
+            mTest = mTest.toBuilder().setConfigure(mTest.getConfigure().toBuilder().addAllParameter(param_list)).build();
+            Log.d(TAG, "Updated test: " + mTest);
+            mStats.updateTest(mTest);
             if (status != 0) {
                 Log.e(TAG, "Init failed");
                 return "";
             }
             headerArray = getHeader();
+            FrameInfo info;
             while (!input_done || !output_done) {
                 try {
                     long timeoutUs = VIDEO_CODEC_WAIT_TIME_US;
@@ -245,11 +292,22 @@ public String start() {
                             continue;
                         }
 
-                        outputBufferSize = encode(yuvData, outputBuffer, width, height, colorSpace, bitdepth);
+                        mFramesAdded++;
+                        long pts = computePresentationTimeUsec(mFramesAdded, mRefFrameTime);
+                        info = mStats.startEncodingFrame(pts, mFramesAdded);
+                        // Let us read the setting in native and force key frame if set here.
+                        // If (for some reason a key frame is not produced it will be updated in the native code
+                        //info.isIFrame(true);
+                        outputBufferSize = encode(yuvData, outputBuffer, info);
+                        Log.d(TAG, "From native pts:" + info.getPts());
                         if (outputBufferSize == 0) {
                             return "Failed to encode frame";
+                        } else if (outputBufferSize == -1) {
+                            return "Encoder not started";
                         }
-                        mFramesAdded++;
+                        // Look at nal type as well, not just key frame?
+                        // To ms?
+                        mStats.stopEncodingFrame(mPts , info.getSize(), info.isIFrame());
                         currentFramePosition += frameSize;
                     } catch (IOException e) {
                         e.printStackTrace();
@@ -266,10 +324,10 @@ public String start() {
                         MediaFormat format = MediaFormat.createVideoFormat(MediaFormat.MIMETYPE_VIDEO_AVC, width, height);
                         format.setInteger(MediaFormat.KEY_WIDTH, width);
                         format.setInteger(MediaFormat.KEY_HEIGHT, height);
-                        format.setInteger(MediaFormat.KEY_BIT_RATE, 800000);
-                        format.setInteger(MediaFormat.KEY_FRAME_RATE, 50);
-                        format.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible);
-                        format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 1);
+                        format.setInteger(MediaFormat.KEY_BIT_RATE, bitrate);
+                        format.setFloat(MediaFormat.KEY_FRAME_RATE, 30);//mFrameRate);
+                        format.setInteger(MediaFormat.KEY_COLOR_FORMAT, 20);//MediaCodecInfoHelper.mapEncappPixFmtToAndroidColorFormat(PixFmt.forNumber(pixelformat)));
+                        format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 3);//iframeinterval);//TODO:
 
                         byte[][] spsPps = extractSpsPps(headerArray);
 
@@ -287,8 +345,9 @@ public String start() {
                     ByteBuffer buffer = ByteBuffer.wrap(outputBuffer);
                     bufferInfo.offset = 0;
                     bufferInfo.size = outputBufferSize;
-                    bufferInfo.presentationTimeUs = computePresentationTimeUsec(mFramesAdded, mRefFrameTime);
+                    bufferInfo.presentationTimeUs = info.getPts();
 
+                    //TODO: we get this from FrameInfo instead
                     boolean isKeyFrame = checkIfKeyFrame(outputBuffer);
                     if (isKeyFrame) bufferInfo.flags = MediaCodec.BUFFER_FLAG_KEY_FRAME;
 

app/src/main/java/com/facebook/encapp/MainActivity.java

@@ -44,16 +44,12 @@
 import java.io.FileInputStream;
 import java.io.FileWriter;
 import java.io.IOException;
-import java.nio.file.FileSystems;
-import java.nio.file.Files;
-import java.nio.file.Path;
 import java.util.Arrays;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Locale;
 import java.util.Stack;
 import java.util.Vector;
-import android.media.MediaCodec;
 
 public class MainActivity extends AppCompatActivity {
     private final static String TAG = "encapp.main";
@@ -649,11 +645,8 @@ private Thread PerformTest(Test test) {
                 }
 
             } else {
-                Log.d(TAG, "[" + test.getCommon().getId() + "] BufferEncoder test");
-                if (test.getConfigure().getCodec().equals("ittiam_x264")){
-                    coder = new BufferX264Encoder(test);
-                } else if (test.getConfigure().getCodec().equals("x264")){
-                    coder = new SwLibEncoder(test);
+                if (test.getConfigure().getCodec().endsWith(".so")){
+                    coder = new CustomEncoder(test);
                 } else {
                     coder = new BufferEncoder(test);
                 }

app/src/main/java/com/facebook/encapp/utils/FrameInfo.java

@@ -5,6 +5,7 @@
 
 public class FrameInfo {
     long mPts;
+    long mDts;
     long mSize;
     long mProcessTime;
     long mStartTime;

app/src/main/java/com/facebook/encapp/utils/Statistics.java

@@ -165,11 +165,12 @@ public void stop() {
         mLoad.stop();
     }
 
-    public void startEncodingFrame(long pts, int originalFrame) {
+    public FrameInfo startEncodingFrame(long pts, int originalFrame) {
         FrameInfo frame = new FrameInfo(pts, originalFrame);
         frame.start();
         mEncodingFrames.add(frame);
         mEncodingProcessingFrames += 1;
+        return frame;
     }
 
     public FrameInfo stopEncodingFrame(long pts, long size, boolean isIFrame) {
@@ -535,6 +536,9 @@ public void writeJSON(Writer writer) throws IOException {
     }
 
 
+    public void updateTest(Test test) {
+        mTest = test;
+    }
     public void addRawFrameFilter(RawFrameFilter filter) {
         mRawFrameFilters.add(filter);
     }

app/src/main/java/com/facebook/encapp/utils/StringParameter.java

@@ -0,0 +1,34 @@
+package com.facebook.encapp.utils;
+
+import com.facebook.encapp.proto.DataValueType;
+import com.facebook.encapp.proto.Parameter;
+
+// The protobuf is using a factory way of constructing the objects. One way of solving creating of
+// parameters would be to use a c protobuf implementation. For now we will use a simpler version where
+// we create a simple object with only strings and convert in java instead.
+public class StringParameter {
+    String mKey = "";
+    String mType = "";
+    String mValue = "";
+
+    public StringParameter(String key, String type, String value) {
+        mKey = key;
+        mType = type;
+        mValue = value;
+    }
+
+    public Parameter getParameter() {
+        if (mType.equals(DataValueType.intType.name())) {
+            return Parameter.newBuilder().setType(DataValueType.intType).setKey(mKey).setValue(mValue).build();
+        } else if (mType.equals(DataValueType.longType.name())) {
+            return Parameter.newBuilder().setType(DataValueType.longType).setKey(mKey).setValue(mValue).build();
+        } else if (mType.equals(DataValueType.floatType.name())) {
+            return Parameter.newBuilder().setType(DataValueType.floatType).setKey(mKey).setValue(mValue).build();
+        } else if (mType.equals(DataValueType.longType.name())) {
+            return Parameter.newBuilder().setType(DataValueType.stringType).setKey(mKey).setValue(mValue).build();
+        }
+
+        // Oh no!
+        return null;
+    }
+}