//service修改
List comments = spiderComments(goodsId, 1);
GoodsInfo goodsInfo =new GoodsInfo(goodsId, goodsName, imgUrl, goodsPrice, comments);
goods.add(goodsInfo);
String jsonStr = JSON.toJSONString(goodsInfo);
logger.info("成功爬取【" + goodsName +"】的基本信息 ");
logger.info(jsonStr);
if (index++ ==9) {
break;
}
}
return goods;
}
private ListspiderComments(String goodId, int frequency) {
List goodComments =new ArrayList<>();
for (int page =0; page < frequency; page++) {
String json = loadJson("http://club.jd.com/productpage/p-" + goodId +"-s-0-t-1-p-" + page +".html");
JSONObject jObj = JSONObject.parseObject(json);
if (jObj !=null) {
JSONArray comments = jObj.getJSONArray("comments");
if (comments !=null && !"".equals(comments)) {
for (int i =0; i < comments.size(); i++) {
JSONObject comment = comments.getJSONObject(i);
String tempContent = comment.getString("content");
if (tempContent !=null && !"".equals(tempContent)) {
System.out.println(tempContent);
goodComments.add(tempContent);
}else {
break;
}
}
}
}else {
break;
}
}
return goodComments;
}
private StringloadJson(String url) {
StringBuilder json =new StringBuilder();
try {
URL urlObject =new URL(url);
URLConnection uc = urlObject.openConnection();
BufferedReader in =new BufferedReader(new InputStreamReader(
uc.getInputStream(), "GBK"));
String inputLine =null;
while ((inputLine = in.readLine()) !=null) {
json.append(inputLine);
}
in.close();
}catch (MalformedURLException e) {
e.printStackTrace();
}catch (IOException e) {
e.printStackTrace();
}
return json.toString();
}
}
//修改handle部分
for (int i =1; i <7; i +=2) {
Map params = Maps.newHashMap();
params.put("keyword", keyword);
params.put("enc", "utf-8");
params.put("wc", keyword);
params.put("page", i +"");
executorService.submit(() -> {
tempGoods=spiderService.spiderData(SysConstant.BASE_URL, params);
bigGoods.addAll(tempGoods);
countDownLatch.countDown();
});
}
网友评论