【十二】【vlc-anroid】視訊影象display展示層模組原始碼分析-OpenGL ES2互動渲染
阿新 • • 發佈:2020-12-19
技術標籤:【音視訊】【vlc-android】vlcopengl
接著第十章節分析。
本章節分析openGL預設展示方式。
// 第1種影象輸出層展示方式
display.c (vlc\modules\video_output\android) line 63 : set_capability("vout display", 260)
display.c (vlc\modules\video_output\android) line 69 : set_capability("vout display", 280)
// 第2種影象輸出層展示方式
display.c (vlc\modules\video_output\opengl) line 51 : set_capability ("vout display", 265)
display.c (vlc\modules\video_output\opengl) line 65 : set_capability ("vout display", 270)
// 第3種影象輸出層展示方式
vout.c (vlc\modules\codec\omxil) line 50 : set_capability("vout display" , 0)
// 第4種影象輸出層展示方式【將YUV資料儲存到檔案中】
yuv.c (vlc\modules\video_output) line 62 : set_capability("vout display", 0)
如上分析可知,vlc android預設先嚐試載入OpenGL (ES2)元件模組的影象展示方式。其優先順序高
1、OpenGL元件模組載入宣告:
// vlc/modules/video_output/android/display.c
vlc_module_begin ()
#if defined (USE_OPENGL_ES2)
// 由【USE_OPENGL_ES2】的定義處可知,該定義若存在則表示當前系統為Apple【IOS】
# define API VLC_OPENGL_ES2
# define MODULE_VARNAME "gles2"
set_shortname (N_("OpenGL ES2"))
set_description (N_("OpenGL for Embedded Systems 2 video output"))
set_capability ("vout display", 265)
set_callbacks (Open, Close)
add_shortcut ("opengles2", "gles2")
add_module ("gles2", "opengl es2", NULL,
GLES2_TEXT, PROVIDER_LONGTEXT, true)
#else
# define API VLC_OPENGL
# define MODULE_VARNAME "gl"
set_shortname (N_("OpenGL"))
set_description (N_("OpenGL video output"))
set_category (CAT_VIDEO)
set_subcategory (SUBCAT_VIDEO_VOUT)
set_capability ("vout display", 270)
set_callbacks (Open, Close)
add_shortcut ("opengl", "gl")
add_module ("gl", "opengl", NULL,
GL_TEXT, PROVIDER_LONGTEXT, true)
#endif
add_glopts ()
vlc_module_end ()
OpenGL元件模組載入初始化入口: vlc/modules/video_output/android/display.c
/**
* Allocates a surface and an OpenGL context for video output.
*/
static int Open (vlc_object_t *obj)
{
vout_display_t *vd = (vout_display_t *)obj;
vout_display_sys_t *sys = malloc (sizeof (*sys));
if (unlikely(sys == NULL))
return VLC_ENOMEM;
sys->gl = NULL;
sys->pool = NULL;
// 初始化Android native window物件,並關聯java層對應物件資訊
// 見第十章節中2.1小節分析
vout_window_t *surface = vout_display_NewWindow (vd, VOUT_WINDOW_TYPE_INVALID);
if (surface == NULL)
{
msg_Err (vd, "parent window not available");
goto error;
}
const char *gl_name = "$" MODULE_VARNAME;
/* VDPAU GL interop works only with GLX. Override the "gl" option to force
* it. */
// 由【USE_OPENGL_ES2】的定義處可知,該定義若存在則表示當前系統為Apple【IOS】
#ifndef USE_OPENGL_ES2
// 不存在該定義,則進入,但由上面設定的為【VOUT_WINDOW_TYPE_INVALID】則不會執行if中
if (surface->type == VOUT_WINDOW_TYPE_XID)
{// 此為window上x11設定
switch (vd->fmt.i_chroma)
{
case VLC_CODEC_VDPAU_VIDEO_444:
case VLC_CODEC_VDPAU_VIDEO_422:
case VLC_CODEC_VDPAU_VIDEO_420:
{
/* Force the option only if it was not previously set */
char *str = var_InheritString(surface, MODULE_VARNAME);
if (str == NULL || str[0] == 0 || strcmp(str, "any") == 0)
gl_name = "glx";
free(str);
break;
}
default:
break;
}
}
#endif
// 建立載入【“opengl”】名的元件模組
// 請參考第十三章節分析 TODO
sys->gl = vlc_gl_Create (surface, API, gl_name);
if (sys->gl == NULL)
goto error;
// 呼叫【gl->resize(gl, w, h)】對應方法設定GL視窗大小
// 但由第十三章節分析可知,該功能在android端不支援
vlc_gl_Resize (sys->gl, vd->cfg->display.width, vd->cfg->display.height);
/* Initialize video display */
const vlc_fourcc_t *spu_chromas;
// 有十三章節分析,該功能讓egl各模組關聯並啟動opengl元件
if (vlc_gl_MakeCurrent (sys->gl))
goto error;
// 建立【vout_display_opengl_t】物件,並載入egl支援的介面API等
sys->vgl = vout_display_opengl_New (&vd->fmt, &spu_chromas, sys->gl,
&vd->cfg->viewpoint);
vlc_gl_ReleaseCurrent (sys->gl);
if (sys->vgl == NULL)
goto error;
vd->sys = sys;
vd->info.has_pictures_invalid = false;
vd->info.subpicture_chromas = spu_chromas;
// 見第2小節分析
vd->pool = Pool;
// 見第3小節分析
vd->prepare = PictureRender;
// 見第4小節分析
vd->display = PictureDisplay;
// 見第5小節分析
vd->control = Control;
return VLC_SUCCESS;
error:
if (sys->gl != NULL)
vlc_gl_Release (sys->gl);
if (surface != NULL)
vout_display_DeleteWindow (vd, surface);
free (sys);
return VLC_EGENERIC;
}
2、Pool實現分析:分配影象緩衝池記憶體
// [vlc/modules/video_output/android/display.c]
static picture_pool_t *Pool (vout_display_t *vd, unsigned count)
{
vout_display_sys_t *sys = vd->sys;
// 注:每次需要訪問EGL介面功能時都需要配套呼叫
// 【vlc_gl_MakeCurrent】和【vlc_gl_ReleaseCurrent】方法
if (!sys->pool && vlc_gl_MakeCurrent (sys->gl) == VLC_SUCCESS)
{// 若此前影象緩衝池buffer為空則建立獲取
sys->pool = vout_display_opengl_GetPool (sys->vgl, count);
vlc_gl_ReleaseCurrent (sys->gl);
}
return sys->pool;
}
// [vlc/modules/video_output/android/vout_helper.c]
picture_pool_t *vout_display_opengl_GetPool(vout_display_opengl_t *vgl, unsigned requested_count)
{
GL_ASSERT_NOERROR();
if (vgl->pool)
return vgl->pool;
opengl_tex_converter_t *tc = vgl->prgm->tc;
requested_count = __MIN(VLCGL_PICTURE_MAX, requested_count);
/* Allocate with tex converter pool callback if it exists */
if (tc->pf_get_pool != NULL)
{// 紋理轉換緩衝池回撥存在則執行進入
vgl->pool = tc->pf_get_pool(tc, requested_count);
if (!vgl->pool)
goto error;
return vgl->pool;
}
// 建立緩衝池陣列
/* Allocate our pictures */
picture_t *picture[VLCGL_PICTURE_MAX] = {NULL, };
unsigned count;
for (count = 0; count < requested_count; count++)
{
// 根據視訊格式資訊建立和初始化影象緩衝物件部分資訊
picture[count] = picture_NewFromFormat(&vgl->fmt);
if (!picture[count])
break;
}
if (count <= 0)
goto error;
// 此處處理為:包裝轉換緩衝池陣列為影象緩衝池物件,
// 對影象緩衝池物件進行擴充套件處理即主要為資料總大小位元組對齊【必須為2的倍數處理】
/* Wrap the pictures into a pool */
vgl->pool = picture_pool_New(count, picture);
if (!vgl->pool)
{
for (unsigned i = 0; i < count; i++)
picture_Release(picture[i]);
goto error;
}
GL_ASSERT_NOERROR();
return vgl->pool;
error:
DelTextures(tc, vgl->texture);
return NULL;
}
3、PictureRender實現分析:影象紋理渲染
// [vlc/modules/video_output/android/display.c]
static void PictureRender (vout_display_t *vd, picture_t *pic, subpicture_t *subpicture)
{
vout_display_sys_t *sys = vd->sys;
if (vlc_gl_MakeCurrent (sys->gl) == VLC_SUCCESS)
{
vout_display_opengl_Prepare (sys->vgl, pic, subpicture);
vlc_gl_ReleaseCurrent (sys->gl);
}
}
// [vlc/modules/video_output/android/vout_helper.c]
int vout_display_opengl_Prepare(vout_display_opengl_t *vgl,
picture_t *picture, subpicture_t *subpicture)
{
GL_ASSERT_NOERROR();
opengl_tex_converter_t *tc = vgl->prgm->tc;
// 紋理轉換請求,將當前紋理更新到待顯示影象上
/* Update the texture */
int ret = tc->pf_update(tc, vgl->texture, vgl->tex_width, vgl->tex_height,
picture, NULL);
if (ret != VLC_SUCCESS)
return ret;
// 上次繪製區域個數
int last_count = vgl->region_count;
gl_region_t *last = vgl->region;
vgl->region_count = 0;
vgl->region = NULL;
tc = vgl->sub_prgm->tc;
if (subpicture) {
// 若有字幕子影象資料則進入
int count = 0;
for (subpicture_region_t *r = subpicture->p_region; r; r = r->p_next)
count++;
vgl->region_count = count;
vgl->region = calloc(count, sizeof(*vgl->region));
// 處理字幕子影象繪製區域資訊,並最後作為紋理處理更新到待顯示影象上
int i = 0;
for (subpicture_region_t *r = subpicture->p_region;
r && ret == VLC_SUCCESS; r = r->p_next, i++) {
gl_region_t *glr = &vgl->region[i];
glr->width = r->fmt.i_visible_width;
glr->height = r->fmt.i_visible_height;
if (!vgl->supports_npot) {
glr->width = GetAlignedSize(glr->width);
glr->height = GetAlignedSize(glr->height);
glr->tex_width = (float) r->fmt.i_visible_width / glr->width;
glr->tex_height = (float) r->fmt.i_visible_height / glr->height;
} else {
glr->tex_width = 1.0;
glr->tex_height = 1.0;
}
glr->alpha = (float)subpicture->i_alpha * r->i_alpha / 255 / 255;
glr->left = 2.0 * (r->i_x ) / subpicture->i_original_picture_width - 1.0;
glr->top = -2.0 * (r->i_y ) / subpicture->i_original_picture_height + 1.0;
glr->right = 2.0 * (r->i_x + r->fmt.i_visible_width ) / subpicture->i_original_picture_width - 1.0;
glr->bottom = -2.0 * (r->i_y + r->fmt.i_visible_height) / subpicture->i_original_picture_height + 1.0;
glr->texture = 0;
/* Try to recycle the textures allocated by the previous
call to this function. */
for (int j = 0; j < last_count; j++) {
if (last[j].texture &&
last[j].width == glr->width &&
last[j].height == glr->height) {
glr->texture = last[j].texture;
memset(&last[j], 0, sizeof(last[j]));
break;
}
}
const size_t pixels_offset =
r->fmt.i_y_offset * r->p_picture->p->i_pitch +
r->fmt.i_x_offset * r->p_picture->p->i_pixel_pitch;
if (!glr->texture)
{
/* Could not recycle a previous texture, generate a new one. */
ret = GenTextures(tc, &glr->width, &glr->height, &glr->texture);
if (ret != VLC_SUCCESS)
continue;
}
/* Use the visible pitch of the region */
r->p_picture->p[0].i_visible_pitch = r->fmt.i_visible_width
* r->p_picture->p[0].i_pixel_pitch;
// 作為紋理處理更新到待顯示影象上
ret = tc->pf_update(tc, &glr->texture, &glr->width, &glr->height,
r->p_picture, &pixels_offset);
}
}
// 顯示完成進行清空紋理記憶體處理
for (int i = 0; i < last_count; i++) {
if (last[i].texture)
DelTextures(tc, &last[i].texture);
}
free(last);
VLC_UNUSED(subpicture);
GL_ASSERT_NOERROR();
return ret;
}
4、PictureDisplay實現分析:影象展示
// [vlc/modules/video_output/android/display.c]
static void PictureDisplay (vout_display_t *vd, picture_t *pic, subpicture_t *subpicture)
{
vout_display_sys_t *sys = vd->sys;
if (vlc_gl_MakeCurrent (sys->gl) == VLC_SUCCESS)
{
vout_display_opengl_Display (sys->vgl, &vd->source);
vlc_gl_ReleaseCurrent (sys->gl);
}
// 顯示完成,釋放清空影象以及字幕記憶體
picture_Release (pic);
if (subpicture != NULL)
subpicture_Delete(subpicture);
}
// [vlc/modules/video_output/android/vout_helper.c]
int vout_display_opengl_Display(vout_display_opengl_t *vgl,
const video_format_t *source)
{
GL_ASSERT_NOERROR();
// 以下功能都基本呼叫了GL / GLES的功能方法指標完成執行功能
/* Why drawing here and not in Render()? Because this way, the
OpenGL providers can call vout_display_opengl_Display to force redraw.
Currently, the OS X provider uses it to get a smooth window resizing */
vgl->vt.Clear(GL_COLOR_BUFFER_BIT);
vgl->vt.UseProgram(vgl->prgm->id);
// 通過縮放尺寸調整輸入視訊格式的顯示區域即調整影象大小來適配EGL的Surface/window大小
if (source->i_x_offset != vgl->last_source.i_x_offset
|| source->i_y_offset != vgl->last_source.i_y_offset
|| source->i_visible_width != vgl->last_source.i_visible_width
|| source->i_visible_height != vgl->last_source.i_visible_height)
{
float left[PICTURE_PLANE_MAX];
float top[PICTURE_PLANE_MAX];
float right[PICTURE_PLANE_MAX];
float bottom[PICTURE_PLANE_MAX];
const opengl_tex_converter_t *tc = vgl->prgm->tc;
for (unsigned j = 0; j < tc->tex_count; j++)
{
float scale_w = (float)tc->texs[j].w.num / tc->texs[j].w.den
/ vgl->tex_width[j];
float scale_h = (float)tc->texs[j].h.num / tc->texs[j].h.den
/ vgl->tex_height[j];
/* Warning: if NPOT is not supported a larger texture is
allocated. This will cause right and bottom coordinates to
land on the edge of two texels with the texels to the
right/bottom uninitialized by the call to
glTexSubImage2D. This might cause a green line to appear on
the right/bottom of the display.
There are two possible solutions:
- Manually mirror the edges of the texture.
- Add a "-1" when computing right and bottom, however the
last row/column might not be displayed at all.
*/
left[j] = (source->i_x_offset + 0 ) * scale_w;
top[j] = (source->i_y_offset + 0 ) * scale_h;
right[j] = (source->i_x_offset + source->i_visible_width ) * scale_w;
bottom[j] = (source->i_y_offset + source->i_visible_height) * scale_h;
}
TextureCropForStereo(vgl, left, top, right, bottom);
int ret = SetupCoords(vgl, left, top, right, bottom);
if (ret != VLC_SUCCESS)
return ret;
vgl->last_source.i_x_offset = source->i_x_offset;
vgl->last_source.i_y_offset = source->i_y_offset;
vgl->last_source.i_visible_width = source->i_visible_width;
vgl->last_source.i_visible_height = source->i_visible_height;
}
// 著色繪製
DrawWithShaders(vgl, vgl->prgm);
// 繪製字幕子影象
/* Draw the subpictures */
// Change the program for overlays
struct prgm *prgm = vgl->sub_prgm;
GLuint program = prgm->id;
opengl_tex_converter_t *tc = prgm->tc;
vgl->vt.UseProgram(program);
// 允許混合子影象
vgl->vt.Enable(GL_BLEND);
vgl->vt.BlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);
/* We need two buffer objects for each region: for vertex and texture coordinates. */
if (2 * vgl->region_count > vgl->subpicture_buffer_object_count) {
if (vgl->subpicture_buffer_object_count > 0)
vgl->vt.DeleteBuffers(vgl->subpicture_buffer_object_count,
vgl->subpicture_buffer_object);
vgl->subpicture_buffer_object_count = 0;
int new_count = 2 * vgl->region_count;
vgl->subpicture_buffer_object = realloc_or_free(vgl->subpicture_buffer_object, new_count * sizeof(GLuint));
if (!vgl->subpicture_buffer_object)
return VLC_ENOMEM;
vgl->subpicture_buffer_object_count = new_count;
vgl->vt.GenBuffers(vgl->subpicture_buffer_object_count,
vgl->subpicture_buffer_object);
}
// 繪製區域【著色等功能命令】
vgl->vt.ActiveTexture(GL_TEXTURE0 + 0);
for (int i = 0; i < vgl->region_count; i++) {
gl_region_t *glr = &vgl->region[i];
const GLfloat vertexCoord[] = {
glr->left, glr->top,
glr->left, glr->bottom,
glr->right, glr->top,
glr->right, glr->bottom,
};
const GLfloat textureCoord[] = {
0.0, 0.0,
0.0, glr->tex_height,
glr->tex_width, 0.0,
glr->tex_width, glr->tex_height,
};
assert(glr->texture != 0);
vgl->vt.BindTexture(tc->tex_target, glr->texture);
tc->pf_prepare_shader(tc, &glr->width, &glr->height, glr->alpha);
vgl->vt.BindBuffer(GL_ARRAY_BUFFER, vgl->subpicture_buffer_object[2 * i]);
vgl->vt.BufferData(GL_ARRAY_BUFFER, sizeof(textureCoord), textureCoord, GL_STATIC_DRAW);
vgl->vt.EnableVertexAttribArray(prgm->aloc.MultiTexCoord[0]);
vgl->vt.VertexAttribPointer(prgm->aloc.MultiTexCoord[0], 2, GL_FLOAT,
0, 0, 0);
vgl->vt.BindBuffer(GL_ARRAY_BUFFER, vgl->subpicture_buffer_object[2 * i + 1]);
vgl->vt.BufferData(GL_ARRAY_BUFFER, sizeof(vertexCoord), vertexCoord, GL_STATIC_DRAW);
vgl->vt.EnableVertexAttribArray(prgm->aloc.VertexPosition);
vgl->vt.VertexAttribPointer(prgm->aloc.VertexPosition, 2, GL_FLOAT,
0, 0, 0);
vgl->vt.UniformMatrix4fv(prgm->uloc.OrientationMatrix, 1, GL_FALSE,
prgm->var.OrientationMatrix);
vgl->vt.UniformMatrix4fv(prgm->uloc.ProjectionMatrix, 1, GL_FALSE,
prgm->var.ProjectionMatrix);
vgl->vt.UniformMatrix4fv(prgm->uloc.ZRotMatrix, 1, GL_FALSE,
prgm->var.ZRotMatrix);
vgl->vt.UniformMatrix4fv(prgm->uloc.YRotMatrix, 1, GL_FALSE,
prgm->var.YRotMatrix);
vgl->vt.UniformMatrix4fv(prgm->uloc.XRotMatrix, 1, GL_FALSE,
prgm->var.XRotMatrix);
vgl->vt.UniformMatrix4fv(prgm->uloc.ZoomMatrix, 1, GL_FALSE,
prgm->var.ZoomMatrix);
vgl->vt.DrawArrays(GL_TRIANGLE_STRIP, 0, 4);
}
vgl->vt.Disable(GL_BLEND);
// 執行了【gl->swap(gl)】交換buffer進行影象顯示
/* Display */
vlc_gl_Swap(vgl->gl);
GL_ASSERT_NOERROR();
return VLC_SUCCESS;
}
5、Control實現分析:顯示控制
// [vlc/modules/video_output/android/display.c]
static int Control (vout_display_t *vd, int query, va_list ap)
{
vout_display_sys_t *sys = vd->sys;
switch (query)
{
#ifndef NDEBUG
case VOUT_DISPLAY_RESET_PICTURES: // not needed
vlc_assert_unreachable();
#endif
case VOUT_DISPLAY_CHANGE_DISPLAY_SIZE:
case VOUT_DISPLAY_CHANGE_DISPLAY_FILLED:
case VOUT_DISPLAY_CHANGE_ZOOM:
{// 影象顯示尺寸變化【大小/全屏/縮放】
vout_display_cfg_t c = *va_arg (ap, const vout_display_cfg_t *);
const video_format_t *src = &vd->source;
vout_display_place_t place;
// 調整紋理反向垂直對齊方式
/* Reverse vertical alignment as the GL tex are Y inverted */
if (c.align.vertical == VOUT_DISPLAY_ALIGN_TOP)
c.align.vertical = VOUT_DISPLAY_ALIGN_BOTTOM;
else if (c.align.vertical == VOUT_DISPLAY_ALIGN_BOTTOM)
c.align.vertical = VOUT_DISPLAY_ALIGN_TOP;
vout_display_PlacePicture (&place, src, &c, false);
vlc_gl_Resize (sys->gl, place.width, place.height);
if (vlc_gl_MakeCurrent (sys->gl) != VLC_SUCCESS)
return VLC_EGENERIC;
// 設定window的寬高比,更新視口矩陣值等
vout_display_opengl_SetWindowAspectRatio(sys->vgl, (float)place.width / place.height);
// 然後呼叫GL / GLES的視窗介面調整其大小
vout_display_opengl_Viewport(sys->vgl, place.x, place.y, place.width, place.height);
vlc_gl_ReleaseCurrent (sys->gl);
return VLC_SUCCESS;
}
case VOUT_DISPLAY_CHANGE_SOURCE_ASPECT:
case VOUT_DISPLAY_CHANGE_SOURCE_CROP:
{// 改變源視訊方向或裁剪源視訊,以下處理和上面類似
const vout_display_cfg_t *cfg = vd->cfg;
vout_display_place_t place;
vout_display_PlacePicture (&place, &vd->source, cfg, false);
if (vlc_gl_MakeCurrent (sys->gl) != VLC_SUCCESS)
return VLC_EGENERIC;
vout_display_opengl_SetWindowAspectRatio(sys->vgl, (float)place.width / place.height);
vout_display_opengl_Viewport(sys->vgl, place.x, place.y, place.width, place.height);
vlc_gl_ReleaseCurrent (sys->gl);
return VLC_SUCCESS;
}
case VOUT_DISPLAY_CHANGE_VIEWPOINT:
// 設定視點引數
return vout_display_opengl_SetViewpoint (sys->vgl,
&va_arg (ap, const vout_display_cfg_t* )->viewpoint);
default:
msg_Err (vd, "Unknown request %d", query);
}
return VLC_EGENERIC;
}
本章節大致分析結束