It seems like the MediaPlayer I'm using in an application has started to hang every 16 or 17 frames.
I am using a GlSurfaceView to render frames played by a MediaPlayer. Everything is rendering fine and at a fast fps rate. The application used to run fine, but since some days, I see the video hanging during at least 500ms every 16 or 17 frames.
The program looks like so, and I am on an Xperia Z1. To make sure there was no regression in the code, I restarted using a tutorial, and this lagging behavior is still there.
Whether I'm using lock (synchronized in Java) or not, or Rendermode.WhenDirty or not, this changes absolutely nothing in the playback.
The program is just an activity and a layout with this custom view. No other code involved. (By the way, the demo is not refactored with C# standard, since this is a temporary snippet, please don't discuss refactoring.)
public class CustomVideoView : GLSurfaceView {
VideoRender mRenderer;
private MediaPlayer mMediaPlayer = null;
private string filePath = "";
private Uri uri = null;
private Context _context;
public CustomVideoView(Context context, IAttributeSet attrs) : base(context, attrs) {
_context = context;
init ();
}
public CustomVideoView(Context context, IAttributeSet attrs, int defStyle) : base(context, attrs) {
_context = context;
init ();
}
public CustomVideoView(Context context) : base(context, null) {
_context = context;
init ();
}
public void init() {
SetEGLContextClientVersion (2);
Holder.SetFormat (Format.Translucent);
SetEGLConfigChooser (8, 8, 8, 8, 16, 0);
filePath = "/storage/sdcard1/download/cat3.mp4";
mRenderer = new VideoRender (_context, mMediaPlayer, filePath, false, this);
SetRenderer (mRenderer);
//RenderMode = Rendermode.WhenDirty;
}
public override void OnResume() {
base.OnResume ();
}
public override void OnPause() {
base.OnPause ();
}
protected override void OnDetachedFromWindow() {
// TODO Auto-generated method stub
base.OnDetachedFromWindow ();
if (mMediaPlayer != null) {
mMediaPlayer.Stop ();
mMediaPlayer.Release ();
}
}
private class VideoRender : Java.Lang.Object, GLSurfaceView.IRenderer, SurfaceTexture.IOnFrameAvailableListener {
private string TAG = "VideoRender";
private const int FLOAT_SIZE_BYTES = 4;
private const int TRIANGLE_VERTICES_DATA_STRIDE_BYTES = 3 * FLOAT_SIZE_BYTES;
private const int TEXTURE_VERTICES_DATA_STRIDE_BYTES = 2 * FLOAT_SIZE_BYTES;
private const int TRIANGLE_VERTICES_DATA_POS_OFFSET = 0;
private const int TRIANGLE_VERTICES_DATA_UV_OFFSET = 0;
private float[] mTriangleVerticesData = { -1.0f, -1.0f, 0, 1.0f,
-1.0f, 0, -1.0f, 1.0f, 0, 1.0f, 1.0f, 0, };
private float[] mTextureVerticesData = { 0.0f, 0.0f, 1.0f, 0.0f, 0.0f, 1.0f, 1.0f, 1.0f };
private FloatBuffer mTriangleVertices;
// extra
private FloatBuffer mTextureVertices;
private string mVertexShader = "uniform mat4 uMVPMatrix;\n"
+ "uniform mat4 uSTMatrix;\n" + "attribute vec4 aPosition;\n"
+ "attribute vec4 aTextureCoord;\n"
+ "varying vec2 vTextureCoord;\n" + "void main() {\n"
+ " gl_Position = uMVPMatrix * aPosition;\n"
+ " vTextureCoord = (uSTMatrix * aTextureCoord).xy;\n" + "}\n";
private string mFragmentShader = "#extension GL_OES_EGL_image_external : require\n"
+ "precision mediump float;\n"
+ "varying vec2 vTextureCoord;\n"
+ "uniform samplerExternalOES sTexture;\n"
+ "void main() {\n"
+ " gl_FragColor = texture2D(sTexture, vTextureCoord);\n"
+ "}\n";
private float[] mMVPMatrix = new float[16];
private float[] mSTMatrix = new float[16];
private float[] projectionMatrix = new float[16];
private int mProgram;
private int mTextureID;
private int muMVPMatrixHandle;
private int muSTMatrixHandle;
private int maPositionHandle;
private int maTextureHandle;
private SurfaceTexture mSurface;
private bool updateSurface = false;
private MediaPlayer mMediaPlayer;
private string _filePath;
private bool _isStreaming = false;
private Context _context;
private CustomVideoView _customVideoView;
private int GL_TEXTURE_EXTERNAL_OES = 0x8D65;
public VideoRender(Context context, MediaPlayer mediaPlayer, string filePath, bool isStreaming, CustomVideoView customVideoView) {
_customVideoView = customVideoView;
_filePath = filePath;
_isStreaming = isStreaming;
_context = context;
mMediaPlayer = mediaPlayer;
mTriangleVertices = ByteBuffer
.AllocateDirect(
mTriangleVerticesData.Length * FLOAT_SIZE_BYTES)
.Order(ByteOrder.NativeOrder()).AsFloatBuffer();
mTriangleVertices.Put(mTriangleVerticesData).Position(0);
// extra
mTextureVertices = ByteBuffer
.AllocateDirect(mTextureVerticesData.Length * FLOAT_SIZE_BYTES)
.Order(ByteOrder.NativeOrder()).AsFloatBuffer();
mTextureVertices.Put(mTextureVerticesData).Position(0);
Android.Opengl.Matrix.SetIdentityM(mSTMatrix, 0);
}
public void OnDrawFrame(Javax.Microedition.Khronos.Opengles.IGL10 glUnused) {
lock (syncLock) {
if (updateSurface) {
mSurface.UpdateTexImage ();
mSurface.GetTransformMatrix (mSTMatrix);
updateSurface = false;
}
}
GLES20.GlClearColor (255.0f, 255.0f, 255.0f, 1.0f);
GLES20.GlClear (GLES20.GlDepthBufferBit
| GLES20.GlColorBufferBit);
GLES20.GlUseProgram (mProgram);
checkGlError ("glUseProgram");
GLES20.GlActiveTexture (GLES20.GlTexture0);
GLES20.GlBindTexture (GL_TEXTURE_EXTERNAL_OES, mTextureID);
mTriangleVertices.Position (TRIANGLE_VERTICES_DATA_POS_OFFSET);
GLES20.GlVertexAttribPointer (maPositionHandle, 3, GLES20.GlFloat,
false, TRIANGLE_VERTICES_DATA_STRIDE_BYTES,
mTriangleVertices);
checkGlError ("glVertexAttribPointer maPosition");
GLES20.GlEnableVertexAttribArray (maPositionHandle);
checkGlError ("glEnableVertexAttribArray maPositionHandle");
mTextureVertices.Position (TRIANGLE_VERTICES_DATA_UV_OFFSET);
GLES20.GlVertexAttribPointer (maTextureHandle, 2, GLES20.GlFloat,
false, TEXTURE_VERTICES_DATA_STRIDE_BYTES, mTextureVertices);
checkGlError ("glVertexAttribPointer maTextureHandle");
GLES20.GlEnableVertexAttribArray (maTextureHandle);
checkGlError ("glEnableVertexAttribArray maTextureHandle");
Android.Opengl.Matrix.SetIdentityM (mMVPMatrix, 0);
GLES20.GlUniformMatrix4fv (muMVPMatrixHandle, 1, false, mMVPMatrix,
0);
GLES20.GlUniformMatrix4fv (muSTMatrixHandle, 1, false, mSTMatrix, 0);
GLES20.GlDrawArrays (GLES20.GlTriangleStrip, 0, 4);
checkGlError ("glDrawArrays");
GLES20.GlFinish ();
}
public void OnSurfaceChanged(Javax.Microedition.Khronos.Opengles.IGL10 glUnused, int width, int height) {
GLES20.GlViewport (0, 0, width, height);
Android.Opengl.Matrix.FrustumM (projectionMatrix, 0, -1.0f, 1.0f, -1.0f, 1.0f,
1.0f, 10.0f);
}
public void OnSurfaceCreated(Javax.Microedition.Khronos.Opengles.IGL10 gl,Javax.Microedition.Khronos.Egl.EGLConfig config) {
mProgram = createProgram (mVertexShader, mFragmentShader);
if (mProgram == 0) {
return;
}
maPositionHandle = GLES20
.GlGetAttribLocation (mProgram, "aPosition");
checkGlError ("glGetAttribLocation aPosition");
if (maPositionHandle == -1) {
throw new RuntimeException (
"Could not get attrib location for aPosition");
}
maTextureHandle = GLES20.GlGetAttribLocation (mProgram,
"aTextureCoord");
checkGlError ("glGetAttribLocation aTextureCoord");
if (maTextureHandle == -1) {
throw new RuntimeException (
"Could not get attrib location for aTextureCoord");
}
muMVPMatrixHandle = GLES20.GlGetUniformLocation (mProgram,
"uMVPMatrix");
checkGlError ("glGetUniformLocation uMVPMatrix");
if (muMVPMatrixHandle == -1) {
throw new RuntimeException (
"Could not get attrib location for uMVPMatrix");
}
muSTMatrixHandle = GLES20.GlGetUniformLocation (mProgram,
"uSTMatrix");
checkGlError ("glGetUniformLocation uSTMatrix");
if (muSTMatrixHandle == -1) {
throw new RuntimeException (
"Could not get attrib location for uSTMatrix");
}
int[] textures = new int[1];
GLES20.GlGenTextures (1, textures, 0);
mTextureID = textures [0];
GLES20.GlBindTexture (GL_TEXTURE_EXTERNAL_OES, mTextureID);
checkGlError ("glBindTexture mTextureID");
GLES20.GlTexParameterf (GL_TEXTURE_EXTERNAL_OES,
GLES20.GlTextureMinFilter, GLES20.GlNearest);
GLES20.GlTexParameterf (GL_TEXTURE_EXTERNAL_OES,
GLES20.GlTextureMagFilter, GLES20.GlLinear);
mSurface = new SurfaceTexture (mTextureID);
// mSurface.SetOnFrameAvailableListener (this);
mSurface.FrameAvailable += (object sender, SurfaceTexture.FrameAvailableEventArgs e) => {
OnFrameAvailable(e.SurfaceTexture);
};
Surface surface = new Surface (mSurface);
mMediaPlayer = new MediaPlayer ();
if (System.IO.File.Exists(_filePath)) {
try {
if (!_isStreaming) {
mMediaPlayer.SetDataSource (_filePath);
} else {
throw new System.NotImplementedException();
//mMediaPlayer.SetDataSource (_context, new Uri.Builder().AppendPath(_filePath));
}
} catch (IllegalArgumentException e) {
// TODO Auto-generated catch block
e.PrintStackTrace ();
} catch (SecurityException e) {
// TODO Auto-generated catch block
e.PrintStackTrace ();
} catch (IllegalStateException e) {
// TODO Auto-generated catch block
e.PrintStackTrace ();
} catch (IOException e) {
// TODO Auto-generated catch block
e.PrintStackTrace ();
}
}
mMediaPlayer.SetSurface (surface);
surface.Release ();
try {
mMediaPlayer.Prepare ();
} catch (IOException t) {
Log.Error (TAG, "media player prepare failed");
}
lock (syncLock) {
updateSurface = false;
}
mMediaPlayer.Start ();
}
private readonly object syncLock = new object ();
public void OnFrameAvailable(SurfaceTexture surface) {
lock (syncLock) {
updateSurface = true;
}
_customVideoView.RequestRender ();
}
private int loadShader(int shaderType, string source) {
int shader = GLES20.GlCreateShader (shaderType);
if (shader != 0) {
GLES20.GlShaderSource (shader, source);
GLES20.GlCompileShader (shader);
int[] compiled = new int[1];
GLES20.GlGetShaderiv (shader, GLES20.GlCompileStatus, compiled, 0);
if (compiled [0] == 0) {
Log.Error (TAG, "Could not compile shader " + shaderType + ":");
Log.Error (TAG, GLES20.GlGetShaderInfoLog (shader));
GLES20.GlDeleteShader (shader);
shader = 0;
}
}
return shader;
}
private int createProgram(string vertexSource, string fragmentSource) {
int vertexShader = loadShader (GLES20.GlVertexShader, vertexSource);
if (vertexShader == 0) {
return 0;
}
int pixelShader = loadShader (GLES20.GlFragmentShader,
fragmentSource);
if (pixelShader == 0) {
return 0;
}
int program = GLES20.GlCreateProgram ();
if (program != 0) {
GLES20.GlAttachShader (program, vertexShader);
checkGlError ("glAttachShader");
GLES20.GlAttachShader (program, pixelShader);
checkGlError ("glAttachShader");
GLES20.GlLinkProgram (program);
int[] linkStatus = new int[1];
GLES20.GlGetProgramiv (program, GLES20.GlLinkStatus,
linkStatus, 0);
if (linkStatus [0] != GLES20.GlTrue) {
Log.Error (TAG, "Could not link program: ");
Log.Error (TAG, GLES20.GlGetProgramInfoLog (program));
GLES20.GlDeleteProgram (program);
program = 0;
}
}
return program;
}
private void checkGlError(string op) {
int error;
while ((error = GLES20.GlGetError ()) != GLES20.GlNoError) {
Log.Error (TAG, op + ": glError " + error);
throw new RuntimeException (op + ": glError " + error);
}
}
}
}
When debugging, here are the calculated FPS overall, how long should a frame be played at this framerate, and how long it actually last. We can see the length of the lag, which finally is around 320ms.
[fps] 15.0627 - norm=66 - cur=44.712
[fps] 15.09347 - norm=66 - cur=45.017
[fps] 15.12472 - norm=66 - cur=44.437
[fps] 15.17346 - norm=65 - cur=32.413
[fps] 15.20476 - norm=65 - cur=44.01
[fps] 15.2337 - norm=65 - cur=45.506
[fps] 15.26154 - norm=65 - cur=46.177
[fps] 14.8815 - norm=67 - cur=334.503
[fps] 14.93206 - norm=66 - cur=29.971
[fps] 14.96286 - norm=66 - cur=44.071
[fps] 14.99153 - norm=66 - cur=45.445
[fps] 15.03538 - norm=66 - cur=34.213
[fps] 15.0695 - norm=66 - cur=41.142
[fps] 15.09754 - norm=66 - cur=44.468
[fps] 15.12501 - norm=66 - cur=45.628
[fps] 15.17139 - norm=65 - cur=31.558
[fps] 15.20057 - norm=65 - cur=44.01
[fps] 15.22785 - norm=65 - cur=45.231
[fps] 15.25471 - norm=65 - cur=45.384
[fps] 15.30203 - norm=65 - cur=30.093
[fps] 15.32664 - norm=65 - cur=46.636
[fps] 15.35203 - norm=65 - cur=45.933
[fps] 15.37996 - norm=65 - cur=44.041
[fps] 15.42686 - norm=64 - cur=29.3
[fps] 15.47278 - norm=64 - cur=30.001
[fps] 15.49799 - norm=64 - cur=45.384
[EDIT]
Rebooting the phone fix the problem. So either the phone itself lack of RAM or storage, or I didn't write the example correctly and the CPU is overworking.
It might be easier to do this with a plain SurfaceView, handling the EGL setup and thread management yourself. There's little value in having a dedicated rendering thread if you're just blitting the video frame. (See Grafika for examples.)
If you do stick with GLSurfaceView, you don't want or need to call glFinish() at the end of onDrawFrame(). That is a synchronous call that will stall your thread until GLES finishes drawing. GLSurfaceView will call eglSwapBuffers() after onDrawFrame() returns.
In any event, a 300ms stall is not likely the result of GLES. Either MediaPlayer is stalling, or something else in the system is waking up and consuming all available CPU resources. You can give systrace a try if you want to debug it further.