diff --git a/pom.xml b/pom.xml index fb6f040..46cdc68 100644 --- a/pom.xml +++ b/pom.xml @@ -14,7 +14,7 @@ 1.8 1.8 4.11 - 0.6.1 + 0.7.3 2.6.2 1.7.21 5.0.1 diff --git a/webporter-collector-zhihu/src/main/java/com/brianway/webporter/collector/zhihu/ZhihuConfiguration.java b/webporter-collector-zhihu/src/main/java/com/brianway/webporter/collector/zhihu/ZhihuConfiguration.java index 4272d2b..b61b049 100644 --- a/webporter-collector-zhihu/src/main/java/com/brianway/webporter/collector/zhihu/ZhihuConfiguration.java +++ b/webporter-collector-zhihu/src/main/java/com/brianway/webporter/collector/zhihu/ZhihuConfiguration.java @@ -1,5 +1,7 @@ package com.brianway.webporter.collector.zhihu; +import java.io.UnsupportedEncodingException; + import com.brianway.webporter.configure.BasicConfiguration; public class ZhihuConfiguration extends BasicConfiguration { @@ -7,11 +9,11 @@ public class ZhihuConfiguration extends BasicConfiguration { public static final String SUBDIR_MEMBER = "member/"; public static final String SUBDIR_FOLLOWEE = "followee/"; - public ZhihuConfiguration(String path) { + public ZhihuConfiguration(String path) throws UnsupportedEncodingException { super(path); } - public ZhihuConfiguration() { + public ZhihuConfiguration() throws Exception{ } @@ -31,7 +33,7 @@ public String getFolloweeDataPath() { return getFolloweePath() + site.getDomain() + "/"; } - public static void main(String[] args) { + public static void main(String[] args) throws Exception{ ZhihuConfiguration configuration = new ZhihuConfiguration(); System.out.println(configuration.getSite()); System.out.println(configuration.getBaseDir()); diff --git a/webporter-collector-zhihu/src/main/java/com/brianway/webporter/collector/zhihu/download/ZhihuFolloweePageProcessor.java b/webporter-collector-zhihu/src/main/java/com/brianway/webporter/collector/zhihu/download/ZhihuFolloweePageProcessor.java index 7327a3c..d0ba7c8 100644 --- a/webporter-collector-zhihu/src/main/java/com/brianway/webporter/collector/zhihu/download/ZhihuFolloweePageProcessor.java +++ b/webporter-collector-zhihu/src/main/java/com/brianway/webporter/collector/zhihu/download/ZhihuFolloweePageProcessor.java @@ -5,14 +5,19 @@ import us.codecraft.webmagic.Page; import us.codecraft.webmagic.Site; import us.codecraft.webmagic.Spider; +import us.codecraft.webmagic.model.annotation.ExtractBy; +import us.codecraft.webmagic.monitor.SpiderMonitor; import us.codecraft.webmagic.processor.PageProcessor; import us.codecraft.webmagic.scheduler.BloomFilterDuplicateRemover; import us.codecraft.webmagic.scheduler.FileCacheQueueScheduler; import us.codecraft.webmagic.selector.Json; +import java.io.UnsupportedEncodingException; import java.util.ArrayList; import java.util.List; +import javax.management.MBeanServer; + /** * Created by brian on 16/11/24. * @@ -21,7 +26,11 @@ */ public class ZhihuFolloweePageProcessor implements PageProcessor { - private Site site = new ZhihuConfiguration().getSite(); + private Site site; + + public ZhihuFolloweePageProcessor() throws Exception{ + this.site = new ZhihuConfiguration().getSite(); + } public void process(Page page) { Json json = page.getJson(); @@ -45,7 +54,7 @@ public Site getSite() { public static String generateFolloweeUrl(String urlToken) { final String URL_TEMPLATE = "https://www.zhihu.com/api/v4/members/%s/followees"; - final String QUERY_PARAMS = "?include=data%5B*%5D.url_token&offset=0&per_page=30&limit=30"; + final String QUERY_PARAMS = "?include=data%5B*%5D.answer_count%2Carticles_count%2Cgender%2Cfollower_count%2Cis_followed%2Cis_following%2Cbadge%5B%3F(type%3Dbest_answerer)%5D.topics&offset=0&limit=20"; String encoded = StringHelper.urlEncode(urlToken); return String.format(URL_TEMPLATE, encoded) + QUERY_PARAMS; @@ -61,9 +70,11 @@ public static List generateFolloweeUrls(List urlTokens) { * 下载关注列表的用户数据,用于提取 url_tokens * @param args 无须其他参数 */ - public static void main(String[] args) { + public static void main(String[] args) throws Exception { String pipelinePath = new ZhihuConfiguration().getFolloweePath(); - int crawlSize = 100_0000; + System.out.println(pipelinePath); + int crawlSize = 100_0000;//下划线使数据更清晰 + System.out.println(crawlSize); Spider.create(new ZhihuFolloweePageProcessor()) .setScheduler(//new QueueScheduler() new FileCacheQueueScheduler(pipelinePath) diff --git a/webporter-collector-zhihu/src/main/java/com/brianway/webporter/collector/zhihu/download/ZhihuMemberPageProcessor.java b/webporter-collector-zhihu/src/main/java/com/brianway/webporter/collector/zhihu/download/ZhihuMemberPageProcessor.java index 1ee8818..5c6d3e8 100644 --- a/webporter-collector-zhihu/src/main/java/com/brianway/webporter/collector/zhihu/download/ZhihuMemberPageProcessor.java +++ b/webporter-collector-zhihu/src/main/java/com/brianway/webporter/collector/zhihu/download/ZhihuMemberPageProcessor.java @@ -17,8 +17,12 @@ */ public class ZhihuMemberPageProcessor implements PageProcessor { - private Site site = new ZhihuConfiguration().getSite(); + private Site site; + public ZhihuMemberPageProcessor() throws Exception{ + new ZhihuConfiguration().getSite(); + } + public void process(Page page) { page.putField(ZhihuPipeline.URL, page.getUrl()); page.putField(ZhihuPipeline.RESPONSE, page.getRawText()); @@ -39,8 +43,9 @@ private static String generateMemberUrl(String urlToken) { /** * 根据提取的 url_token 逐个下载用户的完整信息 * @param args 无须其他参数 + * @throws Exception */ - public static void main(String[] args) { + public static void main(String[] args) throws Exception { ZhihuConfiguration configuration = new ZhihuConfiguration(); String pipelinePath = configuration.getMemberPath(); diff --git a/webporter-collector-zhihu/src/main/java/com/brianway/webporter/collector/zhihu/processor/MemberURLTokenGenerator.java b/webporter-collector-zhihu/src/main/java/com/brianway/webporter/collector/zhihu/processor/MemberURLTokenGenerator.java index b4f1d77..6d6133d 100644 --- a/webporter-collector-zhihu/src/main/java/com/brianway/webporter/collector/zhihu/processor/MemberURLTokenGenerator.java +++ b/webporter-collector-zhihu/src/main/java/com/brianway/webporter/collector/zhihu/processor/MemberURLTokenGenerator.java @@ -33,14 +33,16 @@ public class MemberURLTokenGenerator implements DataProcessor { public final static String URLTOKEN_FILENAME = "url_tokens"; - private final static String DEFAULT_FOLDER = new ZhihuConfiguration().getFolloweeDataPath(); - private final static String DEFAULT_PATH = new ZhihuConfiguration().getFolloweePath() + URLTOKEN_FILENAME; + private static String DEFAULT_FOLDER; + private static String DEFAULT_PATH; private String folder; private String path; - public MemberURLTokenGenerator() { + public MemberURLTokenGenerator() throws Exception { this(DEFAULT_FOLDER, DEFAULT_PATH); + DEFAULT_FOLDER = new ZhihuConfiguration().getFolloweeDataPath(); + DEFAULT_PATH = new ZhihuConfiguration().getFolloweePath() + URLTOKEN_FILENAME; } /** @@ -108,7 +110,7 @@ private Set getURLTokens() { return new HashSet<>(tokens); } - public static void main(String[] args) { + public static void main(String[] args) throws Exception { MemberURLTokenGenerator generator = new MemberURLTokenGenerator(); generator.generateURLTokens().stream() .forEach(System.out::println); diff --git a/webporter-collector-zhihu/src/main/java/com/brianway/webporter/collector/zhihu/processor/ZhihuMemberDataProcessor.java b/webporter-collector-zhihu/src/main/java/com/brianway/webporter/collector/zhihu/processor/ZhihuMemberDataProcessor.java index 96aab46..9296d50 100644 --- a/webporter-collector-zhihu/src/main/java/com/brianway/webporter/collector/zhihu/processor/ZhihuMemberDataProcessor.java +++ b/webporter-collector-zhihu/src/main/java/com/brianway/webporter/collector/zhihu/processor/ZhihuMemberDataProcessor.java @@ -49,7 +49,7 @@ public static String readMember(File inItem) { return followees.size() == 0 ? null : followees.get(0); } - public static void main(String[] args) { + public static void main(String[] args) throws Exception { ZhihuConfiguration configuration = new ZhihuConfiguration(); String folder = configuration.getMemberDataPath(); DataProcessor processor = new ZhihuMemberDataProcessor(); diff --git a/webporter-collector-zhihu/src/main/java/com/brianway/webporter/collector/zhihu/upload/FolloweeUploader.java b/webporter-collector-zhihu/src/main/java/com/brianway/webporter/collector/zhihu/upload/FolloweeUploader.java index 04d31ec..a6f19d4 100644 --- a/webporter-collector-zhihu/src/main/java/com/brianway/webporter/collector/zhihu/upload/FolloweeUploader.java +++ b/webporter-collector-zhihu/src/main/java/com/brianway/webporter/collector/zhihu/upload/FolloweeUploader.java @@ -11,7 +11,7 @@ import java.util.concurrent.TimeUnit; public class FolloweeUploader { - public static void upload() { + public static void upload() throws Exception { String index = "zhihu"; String type = "followee"; ZhihuConfiguration configuration = new ZhihuConfiguration(); @@ -31,7 +31,7 @@ public static void upload() { System.out.println(outPipeline.getBulkProcessor()); } - public static void main(String[] args) { + public static void main(String[] args) throws Exception { FolloweeUploader.upload(); } diff --git a/webporter-collector-zhihu/src/main/java/com/brianway/webporter/collector/zhihu/upload/MemberUploader.java b/webporter-collector-zhihu/src/main/java/com/brianway/webporter/collector/zhihu/upload/MemberUploader.java index c241e73..f0d6906 100644 --- a/webporter-collector-zhihu/src/main/java/com/brianway/webporter/collector/zhihu/upload/MemberUploader.java +++ b/webporter-collector-zhihu/src/main/java/com/brianway/webporter/collector/zhihu/upload/MemberUploader.java @@ -14,7 +14,7 @@ * step 3: 将用户数据导入 Elasticsearch */ public class MemberUploader { - public static void upload() { + public static void upload() throws Exception { String index = "zhihu"; String type = "member"; ZhihuConfiguration configuration = new ZhihuConfiguration(); @@ -34,7 +34,7 @@ public static void upload() { System.out.println(outPipeline.getBulkProcessor()); } - public static void main(String[] args) { + public static void main(String[] args) throws Exception { MemberUploader.upload(); } } diff --git a/webporter-collector-zhihu/src/main/resources/config.json b/webporter-collector-zhihu/src/main/resources/config.json index 6b75d9d..97b610d 100644 --- a/webporter-collector-zhihu/src/main/resources/config.json +++ b/webporter-collector-zhihu/src/main/resources/config.json @@ -2,11 +2,12 @@ "site": { "domain": "www.zhihu.com", "headers": { - "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36", - "authorization": "Your own authorization here." + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.181 Safari/537.36", + "X-UDID": "APCvmFgTjA2PTpjFsTjc15kWKMlBMJJ7Vag=", + "Cookie": "_zap=d35756d6-8530-4512-934a-d4ac74000ee4; __utma=155987696.492403388.1517840990.1517840990.1517840990.1; __utmz=155987696.1517840990.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); d_c0=\"APCvmFgTjA2PTpjFsTjc15kWKMlBMJJ7Vag=|1525518611\"; __DAYU_PP=MMUf3RbMn3ee2mzUNvQJffffffff83b23b7b6aed; q_c1=25af6ae0238e4f39b484eaaf92d1d807|1527508465000|1515585280000; _xsrf=7add93e5-45c4-4733-abdb-d6774ad16ef3; capsion_ticket=\"2|1:0|10:1528022290|14:capsion_ticket|44:YjdjODlhMjRhMWI0NDQzNmI2MjM5YWI0OGFkY2VlZDE=|150deeeec740cbb8c8b21e0b4a41090dc5058800cf4dadf33ddadee2de1bfe55\"; z_c0=\"2|1:0|10:1528022298|4:z_c0|92:Mi4xSU00OUFBQUFBQUFBOEstWVdCT01EU1lBQUFCZ0FsVk5HaE1CWEFCSUg2VmF6RTlCZTQwQTJHc1V0ZTdKR3YzaGtR|2633e75b1b77854a586f118a03068256a1506fb3941446559dea06015d58885e\"; tgw_l7_route=7139e401481ef2f46ce98b22af4f4bed" }, "retryTimes": 3, - "sleepTime": 500 + "sleepTime": 10000 }, - "base_dir": "/Users/brian/todo/zhihu-crawl/" + "base_dir": "F:/KingWang/crawler" } \ No newline at end of file diff --git a/webporter-collector-zhihu/src/test/java/com/brianway/webporter/collector/ZhihuConfigurationTest.java b/webporter-collector-zhihu/src/test/java/com/brianway/webporter/collector/ZhihuConfigurationTest.java index c114c3f..86434f8 100644 --- a/webporter-collector-zhihu/src/test/java/com/brianway/webporter/collector/ZhihuConfigurationTest.java +++ b/webporter-collector-zhihu/src/test/java/com/brianway/webporter/collector/ZhihuConfigurationTest.java @@ -1,6 +1,9 @@ package com.brianway.webporter.collector; import com.brianway.webporter.collector.zhihu.ZhihuConfiguration; + +import java.io.UnsupportedEncodingException; + import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -22,7 +25,7 @@ public void init() { } @Test - public void testConfig() { + public void testConfig() throws Exception { String memberPath = baseDir + ZhihuConfiguration.SUBDIR_MEMBER; ZhihuConfiguration configuration = new ZhihuConfiguration(); Assert.assertEquals(baseDir, configuration.getBaseDir()); @@ -31,7 +34,7 @@ public void testConfig() { } @Test - public void testConfigByPath() { + public void testConfigByPath() throws UnsupportedEncodingException { String followeePath = baseDir + ZhihuConfiguration.SUBDIR_FOLLOWEE; String path = rootDir + "another-config.json"; ZhihuConfiguration configuration = new ZhihuConfiguration(path); diff --git a/webporter-core/src/main/java/com/brianway/webporter/configure/AbstractConfiguration.java b/webporter-core/src/main/java/com/brianway/webporter/configure/AbstractConfiguration.java index f253738..e105348 100644 --- a/webporter-core/src/main/java/com/brianway/webporter/configure/AbstractConfiguration.java +++ b/webporter-core/src/main/java/com/brianway/webporter/configure/AbstractConfiguration.java @@ -1,5 +1,8 @@ package com.brianway.webporter.configure; +import java.io.UnsupportedEncodingException; +import java.net.URLDecoder; + import com.brianway.webporter.util.FileHelper; public abstract class AbstractConfiguration { @@ -9,11 +12,12 @@ public abstract class AbstractConfiguration { protected String config; - protected AbstractConfiguration() { + protected AbstractConfiguration() throws UnsupportedEncodingException { this(DEFAULT_CONFIG_DIR + DEFAULT_CONFIG_FILE); } - protected AbstractConfiguration(String path) { + protected AbstractConfiguration(String path) throws UnsupportedEncodingException { + path = URLDecoder.decode(path,"UTF-8"); config = FileHelper.getRawText(path); resolve(); } diff --git a/webporter-core/src/main/java/com/brianway/webporter/configure/BasicConfiguration.java b/webporter-core/src/main/java/com/brianway/webporter/configure/BasicConfiguration.java index 22f1511..d5dee76 100644 --- a/webporter-core/src/main/java/com/brianway/webporter/configure/BasicConfiguration.java +++ b/webporter-core/src/main/java/com/brianway/webporter/configure/BasicConfiguration.java @@ -1,5 +1,7 @@ package com.brianway.webporter.configure; +import java.io.UnsupportedEncodingException; + import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONObject; import us.codecraft.webmagic.Site; @@ -9,11 +11,11 @@ public class BasicConfiguration extends AbstractConfiguration { protected String baseDir; - public BasicConfiguration(String path) { + public BasicConfiguration(String path) throws UnsupportedEncodingException { super(path); } - public BasicConfiguration() { + public BasicConfiguration() throws Exception{ } @Override diff --git a/webporter-core/src/main/java/com/brianway/webporter/configure/SiteConfiguration.java b/webporter-core/src/main/java/com/brianway/webporter/configure/SiteConfiguration.java index 0e2c07f..69ae8b6 100644 --- a/webporter-core/src/main/java/com/brianway/webporter/configure/SiteConfiguration.java +++ b/webporter-core/src/main/java/com/brianway/webporter/configure/SiteConfiguration.java @@ -1,5 +1,7 @@ package com.brianway.webporter.configure; +import java.io.UnsupportedEncodingException; + import com.alibaba.fastjson.JSON; import us.codecraft.webmagic.Site; @@ -11,11 +13,11 @@ public Site getSite() { return site; } - public SiteConfiguration(String path) { + public SiteConfiguration(String path) throws UnsupportedEncodingException { super(path); } - public SiteConfiguration() { + public SiteConfiguration() throws Exception{ } @@ -23,7 +25,7 @@ protected void resolve() { site = JSON.parseObject(config, Site.class); } - public static void main(String[] args) { + public static void main(String[] args) throws Exception { SiteConfiguration siteConfiguration = new SiteConfiguration(); Site site = siteConfiguration.getSite(); System.out.println(site); diff --git a/webporter-core/src/test/java/com/brianway/webporter/BaseTest.java b/webporter-core/src/test/java/com/brianway/webporter/BaseTest.java index 018bcfa..2b65799 100644 --- a/webporter-core/src/test/java/com/brianway/webporter/BaseTest.java +++ b/webporter-core/src/test/java/com/brianway/webporter/BaseTest.java @@ -1,13 +1,17 @@ package com.brianway.webporter; +import java.io.UnsupportedEncodingException; +import java.net.URLDecoder; + import org.junit.BeforeClass; public class BaseTest { protected static String rootDir; @BeforeClass - public static void init() { + public static void init() throws UnsupportedEncodingException { rootDir = BaseTest.class.getResource("/").getPath(); + rootDir = URLDecoder.decode(rootDir,"UTF-8"); } } diff --git a/webporter-core/src/test/java/com/brianway/webporter/configure/BasicConfigurationTest.java b/webporter-core/src/test/java/com/brianway/webporter/configure/BasicConfigurationTest.java index 70cab65..a99a86f 100644 --- a/webporter-core/src/test/java/com/brianway/webporter/configure/BasicConfigurationTest.java +++ b/webporter-core/src/test/java/com/brianway/webporter/configure/BasicConfigurationTest.java @@ -1,13 +1,16 @@ package com.brianway.webporter.configure; import com.brianway.webporter.BaseTest; + +import java.io.UnsupportedEncodingException; + import org.junit.Assert; import org.junit.Test; import us.codecraft.webmagic.Site; public class BasicConfigurationTest extends BaseTest { @Test - public void testGetConfiguration() { + public void testGetConfiguration() throws UnsupportedEncodingException { String configPath = rootDir + "basic-config.json"; int retryTimes = 3; String domain = "www.zhihu.com"; diff --git a/webporter-core/src/test/java/com/brianway/webporter/configure/SiteConfigurationTest.java b/webporter-core/src/test/java/com/brianway/webporter/configure/SiteConfigurationTest.java index 6ed9875..b3dedd0 100644 --- a/webporter-core/src/test/java/com/brianway/webporter/configure/SiteConfigurationTest.java +++ b/webporter-core/src/test/java/com/brianway/webporter/configure/SiteConfigurationTest.java @@ -7,7 +7,7 @@ public class SiteConfigurationTest { @Test - public void testGetConfiguredSite() { + public void testGetConfiguredSite() throws Exception { SiteConfiguration siteConfiguration = new SiteConfiguration(); Site site = siteConfiguration.getSite(); String key = "AAA";