技術標籤：androidhttpwebview

嗅探網頁視訊的真實地址有很多種方法：

1. 第一種是直接點選播放控制元件的下載 (最簡單直接)但是明顯對m3u8十分無力
2. 用python的you-get庫解析出真實連結，然後用下載軟體下載(可以用於hls或者靜態資源 但是這個網站不一定有特定的規則)
3. 用一些帶嗅探的瀏覽器，在他們載入/播放的時候進行嗅探出真實地址

瀏覽器嗅探資源的原理很直接

以使用了crosswalk核心的app為例，以下是新增crosswalk支援

dependencies {
    compile 'org.xwalk:xwalk_core_library:23.53.589.4'
 

}

allprojects {
    repositories {
        google()
        jcenter()
        maven {
            url 'https://download.01.org/crosswalk/releases/crosswalk/android/maven2'
        }
    }
}

public class MainActivity extends XWalkActivity {

    private XWalkView xWalkWebView;
    private static final String IPHONE_UA =
 
 "Mozilla/5.0 (iPhone; CPU iPhone OS 9_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13B143 Safari/601.1";

    @Override
    protected void onXWalkReady() {
        XWalkPreferences.setValue(XWalkPreferences.REMOTE_DEBUGGING, true);
        XWalkSettings xWalkSettings =
 
 xWalkWebView.getSettings();
        xWalkSettings.setAllowFileAccessFromFileURLs(true);
        xWalkSettings.setAllowUniversalAccessFromFileURLs(true);
        xWalkSettings.setSupportZoom(true);
        xWalkSettings.setBuiltInZoomControls(true);
        xWalkSettings.setJavaScriptEnabled(true);
        xWalkSettings.setUserAgentString(IPHONE_UA);
        xWalkSettings.setJavaScriptCanOpenWindowsAutomatically(true);

        xWalkWebView.requestFocus();

        xWalkWebView.setResourceClient(new MainXWalkResourceClient(xWalkWebView));
        xWalkWebView.setUIClient(new MainXWalkUIClient(xWalkWebView));

        XWalkCookieManager xm = new XWalkCookieManager();
        xm.setAcceptCookie(true);

        xWalkWebView.loadUrl("https://m.youku.com/alipay_video/id_6c09efbfbd48efbfbd48.html?spm=a2hww.12518357.drawer4.dzj1_1", null);
    }

    @Override
    protected void onCreate(Bundle savedInstanceState) {
        super.onCreate(savedInstanceState);
        setContentView(R.layout.activity_main);
        checkPermission();
        Log.i("MainActivity", "onCreate: OnCreate");
        xWalkWebView = (XWalkView) findViewById(R.id.xwalkWebView);
    }


    @Override
    protected void onNewIntent(Intent intent) {
        super.onNewIntent(intent);
        if (isXWalkReady()) xWalkWebView.onNewIntent(intent);
    }

    private void checkPermission() {
        requestPermissions(new String[]{Manifest.permission.ACCESS_WIFI_STATE, Manifest.permission.INTERNET, Manifest.permission.ACCESS_NETWORK_STATE}, 0);
    }

    @Override
    public void onRequestPermissionsResult(int requestCode, @NonNull String[] permissions, @NonNull int[] grantResults) {
        super.onRequestPermissionsResult(requestCode, permissions, grantResults);
        switch (requestCode) {
            case 0: {
                if (grantResults.length > 0
                        && grantResults[0] == PackageManager.PERMISSION_GRANTED) {
                    Toast.makeText(this
                            , "已獲取許可權", Toast.LENGTH_SHORT).show();
                } else {
                    Toast.makeText(this, "您拒絕了", Toast.LENGTH_SHORT).show();
                }
                return;
            }
        }
    }

    class MainXWalkUIClient extends XWalkUIClient {

        public MainXWalkUIClient(XWalkView view) {
            super(view);
        }
    }
}

當我們打開出這個網頁，網頁就會發出很多請求去請求需要的資源，什麼圖片，JavaScript檔案，xhr，視訊，音樂資源，就會被下面這個類進行載入，我們在載入中就對他的每一個請求連結進行再次請求 檢視他響應的Content-Type,如果下面某一個Content-Type正好是響應的Content-Type的子串，那麼這個就很有可能是某個對應的型別，不過m3u8這個型別的還需要進一步解析m3u8的文字格式

 class MainXWalkResourceClient extends XWalkResourceClient {
        private Map<String, List<String>> types = new HashMap<>();

        public MainXWalkResourceClient(XWalkView view) {
            super(view);
            types.put(
                    "m3u8", Arrays.asList("application/octet-stream",
                            "application/vnd.apple.mpegurl",
                            "application/mpegurl",
                            "application/x-mpegurl",
                            "audio/mpegurl",
                            "audio/x-mpegurl"));
            types.put("mp4", Arrays.asList("video/mp4", "application/mp4", "video/h264"));
            types.put("f4v", Arrays.asList("video/x-f4v"));
            types.put("flv", Arrays.asList("video/x-flv"));
        }


        @Override
        public void onLoadStarted(XWalkView view, String url) {
            super.onLoadStarted(view, url);
            Thread thread = new Thread(new Runnable() {
                @Override
                public void run() {
                    try {
                        boolean res = detectUrl(url);
                        Log.d("target", res + "");
                    } catch (Exception e) {
                        e.printStackTrace();
                    }
                }
            });
            thread.start();
        }

        private boolean detectUrl(String url) {
            HttpUtil.Response Response = HttpUtil.getContentType(url);
            if (Response == null) {
//                Log.d("target", "響應為空");
                return false;
            }
            url = Response.getUrl();
            Map<String, List<String>> headerMap = Response.getResponseData();
            if (headerMap == null || !headerMap.containsKey("Content-Type")) {
//                Log.d("target", "header或Content-Type為空");
                return false;
            }

            String type = checkContentType(url,headerMap);
            if (type == null) {
//                Log.d("target", "沒有檢測到是哪種型別");
                return false;
            }

            Log.d("target-hint", "hint " + url + " is " + type+"\t !!!!!!!!!");
            return true;
        }

        public String checkContentType(String url,Map<String, List<String>> data) {
            String contentType="";
            if (data == null) {
                return null;
            }
            if (!data.containsKey("Content-Type")) {
                return null;
            }
            if("mp4".equals(getExtension(url))){
                contentType = "video/mp4";
            }

            contentType = data.get("Content-Type").get(0).toString();
            contentType=contentType.toLowerCase();

            //Log.d("target", "checkContentType: " + contentType);

            Set<Map.Entry<String, List<String>>> entrySet = types.entrySet();
            Iterator<Map.Entry<String, List<String>>> iter = entrySet.iterator();
            while (iter.hasNext()) {
                Map.Entry<String, List<String>> entry = iter.next();
                List<String> strings=entry.getValue();
                for (int i = 0; i < strings.size(); i++) {
                    if (contentType.contains(strings.get(i))){
                        return entry.getKey();
                    }
                }
            }
            return null;
        }
        
        public String getExtension(String fileName){
            if(fileName.lastIndexOf(".")<1){
                return "";
            }
            return fileName.substring(fileName.lastIndexOf(".")+1);
        }
    }

public class HttpUtil {
    private static final Map<String, String> headers = new HashMap<>();

    public final static HostnameVerifier DO_NOT_VERIFY = new HostnameVerifier() {
        public boolean verify(String hostname, SSLSession session) {
            return true;
        }
    };

    {
        headers.put("User-Agent", "Mozilla/5.0 (iPhone; CPU iPhone OS 9_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13B143 Safari/601.1");
    }


    public static HttpUtil.Response getContentType(String url) {
        return getContentType(url, headers);
    }

    public static HttpUtil.Response getContentType(String url, Map<String, String> headers) {
        return getContentType(url, headers, 0);
    }

    public static HttpUtil.Response getContentType(String url, Map<String, String> headers, int redirectCount) {
        HttpURLConnection connection = null;
        try {
            URL UrlObj = new URL(url);
            if (UrlObj.getProtocol().toUpperCase().equals("HTTPS")) {
                TrustManager[] trustAllCerts = new TrustManager[]{
                        new X509TrustManager() {
                            public java.security.cert.X509Certificate[] getAcceptedIssuers() {
                                return new java.security.cert.X509Certificate[]{};
                            }

                            public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException {
                            }

                            public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException {
                            }
                        }
                };
                try {
                    SSLContext sc = SSLContext.getInstance("TLS");
                    sc.init(null, trustAllCerts, new java.security.SecureRandom());
                    HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory());
                } catch (Exception e) {
                    e.printStackTrace();
                }

                HttpsURLConnection https = (HttpsURLConnection) UrlObj.openConnection();
                https.setHostnameVerifier(DO_NOT_VERIFY);
                connection = https;

            } else {
                connection = (HttpURLConnection) UrlObj.openConnection();
            }


            connection.setInstanceFollowRedirects(false);
            connection.setRequestMethod("GET");
            connection.setConnectTimeout(30000);
            connection.setReadTimeout(30000);

            if (headers != null) {
                Set<Map.Entry<String, String>> entrySet = headers.entrySet();
                for (Map.Entry<String, String> entry : entrySet) {
                    connection.setRequestProperty(entry.getKey(), entry.getValue());
                }
            }

            Map<String, List<String>> headerFields = connection.getHeaderFields();

            int responseCode = connection.getResponseCode();
            connection.disconnect();
            if (responseCode == 302) {
                String location = headerFields.get("Location").get(0);
                return getContentType(location, headers, redirectCount + 1);
            } else {
                //Log.d("target", "getContentType: "+!headerFields.isEmpty());
                return new HttpUtil.Response(url, headerFields);
            }

        } catch (IOException e) {
            e.printStackTrace();
            Log.d("target", "error: "+e.getMessage());
        }
        return null;
    }

    public static class Response {
        private String url;
        private Map<String, List<String>> responseData;

        public Response(String url, Map<String, List<String>> responseData) {
            this.url = url;
            this.responseData = responseData;
        }

        public String getUrl() {
            return url;
        }

        public void setUrl(String url) {
            this.url = url;
        }

        public Map<String, List<String>> getResponseData() {
            return responseData;
        }

        public void setResponseData(Map<String, List<String>> responseData) {
            this.responseData = responseData;
        }
    }
}

好看視訊網頁的mp4資源嗅探情況：

優酷m3u8的嗅探情況：

愛奇藝實測不行