Go语言视频零基础入门到精通

五菱

暑假再做一个项目，前提是要获取网站的编码方式！！我开始用的上次编写中文网页分类程序的获取源码，最后我在做项目的时候经常出现获取的编码方式不对！！最后我就把程序的改进了！！现在虽然不能保证100%的对，但是95%都是可以保

2. package spider;
3. import java.io.BufferedReader;
4. import java.io.InputStream;
5. import java.io.InputStreamReader;
6. import java.net.HttpURLConnection;
7. import java.net.URL;
8. import java.util.regex.Matcher;
9. import java.util.regex.Pattern;
10. import org.htmlparser.*;
11. import org.htmlparser.filters.*;
12. import org.htmlparser.util.*;
13. import org.htmlparser.tags.MetaTag;
14. public class getcharset {
15. private String URL = null;
16. private String defaultencode = "gbk";
17. /**
18. *
19. * @param url the web you are going to deal
20. */
21. public getcharset(String url) {
22. URL = url;
23. }
24. /**
25. * method getecode just get the webpage's charset
26. * @return
27. */
28. public String getecode() {
29. try {
30. URL url = new URL(URL);
31. HttpURLConnection conn = (HttpURLConnection) url.openConnection();
32. conn.setConnectTimeout(10000);
33. conn.setReadTimeout(4000);
34. conn.connect();
35. InputStream is = conn.getInputStream();
36. BufferedReader br = new BufferedReader(new InputStreamReader(is));
37. String result = "";
38. String content = "";
39. while ((content = br.readLine()) != null) {
40. result = result + content.toLowerCase();
41. if (content.toLowerCase().contains("</head>")) {
42. break;
43. }
44. }
45. //System.out.println(result);
46. return getencode(result);
47. } catch (Exception ev) {
48. //System.out.println("1==================" + ev.getMessage());
49. return defaultencode; }
50. }
51. private String getencode(String resource) {
52. try {
53. String encode = "";
54. Parser parser = new Parser(resource);
55. NodeFilter filtertag = new TagNameFilter("meta");
56. NodeFilter filterA = new HasAttributeFilter("http-equiv", "content-type");
57. NodeFilter filter = new AndFilter(filtertag, filterA);
58. NodeList nodes = parser.extractAllNodesThatMatch(filter);
59. MetaTag charset = null;
60. if (nodes.size() != 0) {
61. for (int i = 0; i < nodes.size(); i++) {
62. charset = (MetaTag) nodes.elementAt(i);
63. }
64. String content = charset.getMetaContent();
65. if (content.contains("charset")) {
66. int flag2 = content.indexOf("=");
67. encode = content.substring(flag2 + 1);
68. parser = null;
69. if (encode != null || !encode.equals("")) {
70. return encode;
71. } else {
72. return defaultencode;
73. }
74. } else {
75. content = charset.toHtml();
76. String[] thecode = content.split("charset");
77. String code = thecode[1].replaceAll("[=/\">]*", "");
78. return code;
79. }
80. } else {
81. return defaultencode;
82. }
83. } catch (Exception e) {
84. //System.out.println("2==================" + e.getMessage());
85. String codeing = finalGet(resource);
86. if (!codeing.equals("")) {
87. return codeing.trim();
88. } else {
89. return defaultencode;
90. }
91. }
92. }
93. private String finalGet(String resource) {
94. String result = "";
95. String code = "";
96. String reg = "<meta http-equiv=[\"]{0,1}content-type[\"]{0,1}";
97. Matcher m = Pattern.compile(reg).matcher(resource.toLowerCase());
98. while (m.find()) {
99. result = result + resource.substring(m.start());
100. }
101. int flag = result.indexOf(">");
102. //System.out.println(resource+"========");
103. String[] thecode = result.substring(0, flag).toLowerCase().replaceAll(reg, "").split("charset");
104. // if (thecode[0].toLowerCase().contains("charset=")) {
105. // code = thecode[0].replaceAll("(charset=)", "");
106. // } else {
107. code = thecode[1].replaceAll("[=/\"]*", "");
108. // }
109. // System.out.println(code+"======");
110. return code;
111. }
112. }

复制代码