Commit 074bac95 authored by litianle's avatar litianle

爬虫commit

parent 653e8c32
全国主要KA连锁和便利店连锁 爬虫项目
\ No newline at end of file
package com.deeprec.crawler.entry;
import cn.hutool.json.JSONObject;
import lombok.Data;
import lombok.EqualsAndHashCode;
......@@ -20,6 +21,14 @@ public class CrawlerStore extends BaseEntity {
private String type;
private String shopAddr;
private String address;
private String email;
private JSONObject extInfo;
public String getExtInfo() {
if (extInfo == null) return "{}";
return extInfo.toString();
}
}
......@@ -3,18 +3,27 @@ package com.deeprec.crawler.mapper;
import cn.hutool.db.Db;
import cn.hutool.db.Entity;
import com.deeprec.crawler.entry.CrawlerStore;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import java.sql.SQLException;
@Slf4j
public class InsertDB {
private static InsertDB insertDB;
private InsertDB() {}
public static InsertDB getDB() {
return new InsertDB();
if (insertDB == null) {
insertDB = new InsertDB();
}
return insertDB;
}
public boolean insert(CrawlerStore store) throws SQLException {
Entity entity = Entity.create("crawler_store").parseBean(store, true, true);
@SneakyThrows
public boolean insert(CrawlerStore store) {
Entity entity = Entity.create("crawler_store")
.parseBean(store, true, true);
return Db.use().insert(entity) == 0;
}
}
package com.deeprec.crawler.modules.eebbk.crawler;
import cn.hutool.http.HttpUtil;
import cn.hutool.json.JSONObject;
import com.deeprec.crawler.modules.eebbk.dto.AuthorizeStationParam;
import com.deeprec.crawler.modules.eebbk.dto.AuthorizeStationResult;
import com.deeprec.crawler.entry.CrawlerStore;
import com.deeprec.crawler.mapper.InsertDB;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import java.util.List;
import java.util.concurrent.TimeUnit;
@Slf4j
public class AuthorizeStationCrawler {
private static final String SK_NAME = "步步高";
private static final String URL = "https://www.eebbk.com/html/pc/service/service.html";
@SneakyThrows
public static void main(String[] args) {
// 授权服务站点
String station = HttpUtil.createGet("https://service-center.okii.com/api/official-site/area")
.form("siteType", "station")
.execute().body();
List<AuthorizeStationParam> directArray = new JSONObject(station)
.getJSONArray("data")
.toList(AuthorizeStationParam.class);
for (AuthorizeStationParam param : directArray) {
String request = HttpUtil.createGet("https://service-center.okii.com/api/official-site/sites")
.form("pageIndex", 1)
.form("pageSize", 1000)
.form("siteType", "station")
.form("officialProvince", param.getProvince())
.form("officialCity", param.getCity())
.form("officialCounty", param.getCounty())
.execute().body();
List<AuthorizeStationResult> authorizeStationResults = new JSONObject(request)
.getJSONArray("data")
.toList(AuthorizeStationResult.class);
for (AuthorizeStationResult result : authorizeStationResults) {
CrawlerStore crawlerStore = new CrawlerStore()
.setSkName(SK_NAME)
.setCrawlerUrl(URL)
.setType("授权服务站")
.setName(result.getSiteName())
.setPhone(result.getTel())
.setAddress(result.getAddress())
.setExtInfo(new JSONObject()
.set("province", result.getOfficialProvince())
.set("restTime", result.getRestTime())
.set("city", result.getOfficialCity())
.set("authorizationCode", result.getAuthorizationCode())
.set("showNameBbk", result.getShowNameBbk())
.set("businessTime", result.getBusinessTime())
.set("county", result.getOfficialCounty())
.set("county", result.getShowNameXtc()));
log.info("区域: {},{}", result.getAddress(), crawlerStore);
InsertDB.getDB().insert(crawlerStore);
}
TimeUnit.MILLISECONDS.sleep(300);
}
log.info("授权服务站点全部插入完成");
}
}
package com.deeprec.crawler.modules.eebbk.crawler;
import cn.hutool.http.HttpUtil;
import cn.hutool.json.JSONObject;
import com.deeprec.crawler.modules.eebbk.dto.CustomerServiceParam;
import com.deeprec.crawler.modules.eebbk.dto.CustomerServiceResult;
import com.deeprec.crawler.entry.CrawlerStore;
import com.deeprec.crawler.mapper.InsertDB;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import java.util.List;
import java.util.concurrent.TimeUnit;
@Slf4j
public class CustomerServiceCrawler {
private static final String SK_NAME = "步步高";
private static final String URL = "https://www.eebbk.com/html/pc/service/service.html";
@SneakyThrows
public static void main(String[] args) {
String direct = HttpUtil.createGet("https://service-center.okii.com/api/official-site/area")
.form("siteType", "direct")
.execute().body();
List<CustomerServiceParam> directArray = new JSONObject(direct).getJSONArray("data")
.toList(CustomerServiceParam.class);
for (CustomerServiceParam param : directArray) {
String request = HttpUtil.createGet("https://service-center.okii.com/api/official-site/sites")
.form("pageIndex", 1)
.form("pageSize", 1000)
.form("siteType", "direct")
.form("officialProvince", param.getProvince())
.form("officialCity", param.getCity())
.execute().body();
List<CustomerServiceResult> salesOutlets = new JSONObject(request)
.getJSONArray("data")
.toList(CustomerServiceResult.class);
for (CustomerServiceResult result : salesOutlets) {
CrawlerStore crawlerStore = new CrawlerStore()
.setSkName(SK_NAME)
.setCrawlerUrl(URL)
.setType("客服中心")
.setName(result.getSiteName())
.setPhone(result.getTel())
.setAddress(result.getAddress())
.setExtInfo(new JSONObject()
.set("province", result.getOfficialProvince())
.set("city", result.getOfficialCity())
.set("county", result.getOfficialCounty())
.set("openingHours", result.getBusinessTime())
.set("restTime", result.getRestTime())
.set("showNameXtc", result.getShowNameXtc())
.set("showNameBbk", result.getShowNameBbk()));
log.info("区域: {},{}", result.getAddress(), crawlerStore);
InsertDB.getDB().insert(crawlerStore);
}
TimeUnit.MILLISECONDS.sleep(300);
}
log.info("客服中心全部插入完成");
}
}
package com.deeprec.crawler.modules.eebbk.crawler;
import cn.hutool.http.HttpUtil;
import cn.hutool.json.JSONObject;
import com.deeprec.crawler.modules.eebbk.dto.SalesOutletsParam;
import com.deeprec.crawler.modules.eebbk.dto.SalesOutletsResult;
import com.deeprec.crawler.entry.CrawlerStore;
import com.deeprec.crawler.mapper.InsertDB;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import java.util.List;
import java.util.concurrent.TimeUnit;
@Slf4j
public class SalesOutlesEebbkCrawler {
private static final String SK_NAME = "步步高";
private static final String URL = "https://www.eebbk.com/html/pc/service/service.html";
@SneakyThrows
public static void main(String[] args) {
String salesOutletsParam = HttpUtil.createGet("https://scrmapis.okii.com/area").execute().body();
List<SalesOutletsParam> areas = new JSONObject(salesOutletsParam)
.getJSONArray("data")
.toList(SalesOutletsParam.class);
for (SalesOutletsParam area : areas) {
String request = HttpUtil.createGet("https://www.eebbk.com/api/network/sales")
.form("brand", "eebbk")
.form("pageIndex", 1)
.form("province", area.getProvince())
.form("city", area.getCity())
.form("county", area.getCounty())
.execute().body();
List<SalesOutletsResult> salesOutlets = new JSONObject(request)
.getJSONArray("data")
.toList(SalesOutletsResult.class);
for (SalesOutletsResult e : salesOutlets) {
CrawlerStore crawlerStore = new CrawlerStore()
.setSkName(SK_NAME)
.setCrawlerUrl(URL)
.setName(e.getTitle())
.setPhone(e.getTel())
.setAddress(e.getAddress())
.setType("销售网点")
.setExtInfo(new JSONObject()
.set("province", e.getProvince())
.set("city", e.getCity())
.set("county", e.getCounty())
.set("openingHours", e.getOpeningHours()));
log.info("区域: {},{}", e.getAddress(), crawlerStore);
InsertDB.getDB().insert(crawlerStore);
}
TimeUnit.MILLISECONDS.sleep(300);
}
log.info("销售网点全部插入完成");
}
}
package com.deeprec.crawler.modules.eebbk.dto;
import lombok.Data;
// 步步高 授权站点查询结构
@Data
public class AuthorizeStationParam {
String province;
String city;
String county;
}
package com.deeprec.crawler.modules.eebbk.dto;
import lombok.Data;
@Data
public class AuthorizeStationResult {
String officialProvince;
String restTime;
String address;
String officialCity;
String authorizationCode;
String showNameBbk;
String siteName;
String businessTime;
String officialCounty;
String tel;
String showNameXtc;
}
package com.deeprec.crawler.modules.eebbk.dto;
import lombok.Data;
// 步步高 客服中心查询结构
@Data
public class CustomerServiceParam {
String province;
String city;
}
package com.deeprec.crawler.modules.eebbk.dto;
import lombok.Data;
@Data
public class CustomerServiceResult {
String siteName;
String tel;
String address;
String officialProvince;
String officialCity;
String officialCounty;
String wechatWorkId;
String businessTime;
String restTime;
String notice;
String showNameXtc;
String showNameBbk;
}
package com.deeprec.crawler.modules.eebbk.dto;
import lombok.Data;
// 步步高 销售网点区域查询结构
@Data
public class SalesOutletsParam {
String province;
String city;
String county;
}
package com.deeprec.crawler.modules.eebbk.dto;
import lombok.Data;
// 步步高 销售网点区域结果返回
@Data
public class SalesOutletsResult {
String title;
String province;
String city;
String county;
String address;
String tel;
String openingHours;
}
package com.deeprec.crawler.modules.familyMart.crawler;
import cn.hutool.http.HttpUtil;
import cn.hutool.json.JSONObject;
import com.deeprec.crawler.entry.CrawlerStore;
import com.deeprec.crawler.mapper.InsertDB;
import com.deeprec.crawler.modules.familyMart.dto.Store;
import lombok.extern.slf4j.Slf4j;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@Slf4j
public class FamilyMartCrawler {
private static final String SK_NAME = "全家";
private static final String URL = "http://www.familymart.com.cn/store";
private static final Map<String, String> citys = new LinkedHashMap<>();
/**
* /store/Search?cid=1&page=1 POST请求
* cid写死在页面上 上海、苏州、深圳、广州、杭州、成都 分别是1至6
* 无锡、北京、东莞、嘉兴为10至13
* 返回的结果为
* {
* "city": "上海",
* "mapmsg": [,...], --> 所有结果一共3045条数据
* "msg": [,...], --> 当前分页的10条数据
* "nowpage": "1",
* "preferen": false,
* "totalcount": "3045",
* "totalpage": 305
* }
* 直接使用返回的mapmsg即可
*/
public static void main(String[] args) {
String response = HttpUtil.createPost("http://www.familymart.com.cn/store").execute().body();
Document document = Jsoup.parse(response);
Elements elements = document.getElementsByClass("searchbox-sc");
String html = elements.html();
Elements options = Jsoup.parse(html).getElementsByTag("option");
for (Element option : options) {
citys.put(option.text(), option.val());
}
citys.forEach((cityName, cid) -> {
String station = HttpUtil.createPost("http://www.familymart.com.cn/store/Search")
.form("cid", cid)
.form("aid")
.form("street")
.form("address")
.form("service")
.form("name")
.form("page", 1)
.execute().body();
List<Store> stores = new JSONObject(station).getJSONArray("mapmsg").toList(Store.class);
stores.forEach(e -> {
CrawlerStore crawlerStore = new CrawlerStore().setCrawlerUrl(URL)
.setSkName(SK_NAME).setPhone(e.getTelphone())
.setType(cityName)
.setName(e.getName())
.setAddress(e.getStreet())
.setExtInfo(new JSONObject().set("createtime", e.getCreatetime())
.set("ts", e.getTs())
);
InsertDB.getDB().insert(crawlerStore);
log.info("区域: {},{}", crawlerStore.getAddress(), crawlerStore);
});
});
log.info("全家便利店全部插入完成");
}
}
package com.deeprec.crawler.modules.familyMart.dto;
import lombok.Data;
@Data
public class Store {
String createtime;
String uname;
String last_sync_time;
String zm;
String street;
String service;
String telphone;
String name;
String cid;
String ts;
}
package com.deeprec.crawler.modules.maidelong.crawler;
import cn.hutool.http.HttpUtil;
import cn.hutool.json.JSONArray;
import cn.hutool.json.JSONObject;
import com.deeprec.crawler.entry.CrawlerStore;
import com.deeprec.crawler.modules.maidelong.dto.Store;
import com.deeprec.crawler.mapper.InsertDB;
import lombok.extern.slf4j.Slf4j;
import java.util.List;
@Slf4j
public class MaidelongCrawler {
private static final String SK_NAME = "麦德龙";
private static final String URL = "https://www.maidelong.com/newmetro-frontend-web/#/stores/165";
public static void main(String[] args) {
String response = HttpUtil.createGet("https://www.maidelong.com/newmetro-frontend-api/stores/getStoresInfo")
.form("lang", "zh-cn")
.form("curLat", "")
.form("curLng", "")
.form("maxDistance", "100000000")
.form("limit", "999")
.execute().body();
JSONArray jsonArray = new JSONObject(response).getJSONObject("data").getJSONArray("storesData");
List<Store> stores = jsonArray.toList(Store.class);
stores.forEach(e -> {
CrawlerStore crawlerStore = new CrawlerStore()
.setCrawlerUrl(URL)
.setSkName(SK_NAME)
.setName(e.getName())
.setType(e.getRegionName())
.setEmail(e.getEmail())
.setAddress(e.getAddress())
.setPhone(e.getTelephone())
.setExtInfo(new JSONObject().set("manger", e.getManagerCN())
.set("openingHours", e.getOpeningHoursCN())
.set("postCode", e.getPostcode()));
InsertDB.getDB().insert(crawlerStore);
log.info(e.getName() + " - " + e.getRegionName() + " -> 导入");
});
log.info(SK_NAME + "全部导入完成!");
}
}
package com.deeprec.crawler.modules.maidelong.dto;
import lombok.Data;
@Data
public class Store {
String regionName;
String openingHoursCN;
String managerCN;
String email;
String address;
String postcode;
String telephone;
String name;
}
package com.deeprec.crawler.suguo;
package com.deeprec.crawler.modules.suguo;
import cn.hutool.http.HttpUtil;
import com.deeprec.crawler.entry.CrawlerStore;
......@@ -61,7 +61,7 @@ public class SuguoCrawler {
.setSkName(SK_NAME)
.setName(shopName)
.setType(shopType)
.setShopAddr(shopAddr)
.setAddress(shopAddr)
.setPhone(shopPhone);
InsertDB.getDB().insert(crawlerStore);
}
......
package com.deeprec.crawler.modules.yonghui;
import cn.hutool.http.HttpUtil;
import cn.hutool.json.JSONObject;
import com.deeprec.crawler.entry.CrawlerStore;
import com.deeprec.crawler.mapper.InsertDB;
import lombok.extern.slf4j.Slf4j;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
@Slf4j
public class YonghuiCrawler {
private static final String SK_NAME = "永辉超市";
private static final String URL = "https://www.yonghui.com.cn/store_list";
public static void main(String[] args) {
int page = 0;
while (true) {
String station = HttpUtil.createGet("https://www.yonghui.com.cn/mapi/proajax")
.form("act", 2)
.form("pid", 0)
.form("ctlgid", 0)
.form("keyword")
.form("p", page++)
.form("rnd", "742.4132773474001")
.execute().body();
String isComplete = station.substring(5);
if ("暂无数据".equals(isComplete)) {
break;
}
Document document = Jsoup.parse(station);
Elements elements = document.getElementsByClass("storeList");
for (Element element : elements) {
Elements lis = element.getElementsByTag("li");
for (Element li : lis) {
String name = li.getElementsByTag("h1").text();
Elements span = li.getElementsByTag("span");
String[] split = span.html().split("<br>");
split[split.length - 1] = "";
String address = split[0].substring(5);
String phone = split[1].substring(5);
String openingTime = split[2].substring(5);
String busRoute = split[3].substring(5) + split[4];
CrawlerStore crawlerStore = new CrawlerStore()
.setSkName(SK_NAME)
.setCrawlerUrl(URL)
.setType("已开业")
.setName(name)
.setPhone(phone)
.setAddress(address)
.setExtInfo(new JSONObject()
.set("openingTime", openingTime)
.set("busRoute", busRoute));
InsertDB.getDB().insert(crawlerStore);
log.info(name + " - " + address + " -> 导入");
}
}
}
log.info(SK_NAME + "已开业店铺导入完成!");
}
}
package com.deeprec.crawler.modules.yonghui;
import cn.hutool.http.HttpUtil;
import cn.hutool.json.JSONObject;
import com.deeprec.crawler.entry.CrawlerStore;
import com.deeprec.crawler.mapper.InsertDB;
import lombok.extern.slf4j.Slf4j;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
@Slf4j
public class YonghuiOfPrepareCrawler {
private static final String SK_NAME = "永辉超市";
private static final String URL = "https://www.yonghui.com.cn/store_list";
public static void main(String[] args) {
int page = 0;
while (true) {
String station = HttpUtil.createGet("https://www.yonghui.com.cn/mapi/proajax")
.form("act", 3)
.form("pid", 0)
.form("ctlgid", 0)
.form("keyword")
.form("p", page++)
.form("rnd", "742.4132773474001")
.execute().body();
String isComplete = station.substring(4);
if ("暂无数据".equals(isComplete)) {
break;
}
Document document = Jsoup.parse(station);
Elements elements = document.getElementsByClass("storeList");
for (Element element : elements) {
Elements lis = element.getElementsByTag("li");
for (Element li : lis) {
String name = li.getElementsByTag("h1").text();
Elements span = li.getElementsByTag("span");
String[] split = span.html().split("<br>");
String address = split[0].substring(5);
CrawlerStore crawlerStore = new CrawlerStore()
.setSkName(SK_NAME)
.setCrawlerUrl(URL)
.setType("筹备中")
.setName(name)
.setExtInfo(new JSONObject().set("openingTime", split[1].substring(5)));
InsertDB.getDB().insert(crawlerStore);
log.info(name + " - " + address + " -> 导入");
}
}
}
log.info(SK_NAME + "已开业店铺导入完成!");
}
}
package com.deeprec.crawler.utils;
import cn.hutool.aop.aspects.SimpleAspect;
import java.lang.reflect.Method;
import java.util.List;
public class AspectUtil extends SimpleAspect {
private List<Object> list;
@Override
public boolean before(Object target, Method method, Object[] args) {
return true;
}
@Override
public boolean after(Object target, Method method, Object[] args, Object returnVal) {
return true;
}
@Override
public boolean afterException(Object target, Method method, Object[] args, Throwable e) {
return true;
}
}
......@@ -18,14 +18,13 @@ public class CrawlerUtil {
if (param != null)
bindings.putAll(param);
Object result = JSFunction.eval(bindings);
ScriptObjectMirror scpt = (ScriptObjectMirror) result;
ScriptObjectMirror scpt = (ScriptObjectMirror) JSFunction.eval(bindings);
List<Object> list = new ArrayList<>(scpt.values());
List<String> r = new ArrayList<>();
for (Object o : list) {
r.add(o.toString());
}
return r ;
return r;
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment