Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
C
crawler_store
Project
Project
Details
Activity
Releases
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
litianle
crawler_store
Commits
074bac95
Commit
074bac95
authored
Mar 18, 2022
by
litianle
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
爬虫commit
parent
653e8c32
Changes
21
Show whitespace changes
Inline
Side-by-side
Showing
21 changed files
with
613 additions
and
12 deletions
+613
-12
README.md
README.md
+1
-0
CrawlerStore.java
src/main/java/com/deeprec/crawler/entry/CrawlerStore.java
+10
-1
InsertDB.java
src/main/java/com/deeprec/crawler/mapper/InsertDB.java
+15
-6
AuthorizeStationCrawler.java
...rawler/modules/eebbk/crawler/AuthorizeStationCrawler.java
+69
-0
CustomerServiceCrawler.java
...crawler/modules/eebbk/crawler/CustomerServiceCrawler.java
+63
-0
SalesOutlesEebbkCrawler.java
...rawler/modules/eebbk/crawler/SalesOutlesEebbkCrawler.java
+60
-0
AuthorizeStationParam.java
...prec/crawler/modules/eebbk/dto/AuthorizeStationParam.java
+11
-0
AuthorizeStationResult.java
...rec/crawler/modules/eebbk/dto/AuthorizeStationResult.java
+18
-0
CustomerServiceParam.java
...eprec/crawler/modules/eebbk/dto/CustomerServiceParam.java
+11
-0
CustomerServiceResult.java
...prec/crawler/modules/eebbk/dto/CustomerServiceResult.java
+20
-0
SalesOutletsParam.java
.../deeprec/crawler/modules/eebbk/dto/SalesOutletsParam.java
+11
-0
SalesOutletsResult.java
...deeprec/crawler/modules/eebbk/dto/SalesOutletsResult.java
+15
-0
FamilyMartCrawler.java
...crawler/modules/familyMart/crawler/FamilyMartCrawler.java
+78
-0
Store.java
...ava/com/deeprec/crawler/modules/familyMart/dto/Store.java
+18
-0
MaidelongCrawler.java
...c/crawler/modules/maidelong/crawler/MaidelongCrawler.java
+47
-0
Store.java
...java/com/deeprec/crawler/modules/maidelong/dto/Store.java
+15
-0
SuguoCrawler.java
.../java/com/deeprec/crawler/modules/suguo/SuguoCrawler.java
+2
-2
YonghuiCrawler.java
...a/com/deeprec/crawler/modules/yonghui/YonghuiCrawler.java
+65
-0
YonghuiOfPrepareCrawler.java
...prec/crawler/modules/yonghui/YonghuiOfPrepareCrawler.java
+56
-0
AspectUtil.java
src/main/java/com/deeprec/crawler/utils/AspectUtil.java
+26
-0
CrawlerUtil.java
src/main/java/com/deeprec/crawler/utils/CrawlerUtil.java
+2
-3
No files found.
README.md
0 → 100644
View file @
074bac95
全国主要KA连锁和便利店连锁 爬虫项目
\ No newline at end of file
src/main/java/com/deeprec/crawler/entry/CrawlerStore.java
View file @
074bac95
package
com
.
deeprec
.
crawler
.
entry
;
import
cn.hutool.json.JSONObject
;
import
lombok.Data
;
import
lombok.EqualsAndHashCode
;
...
...
@@ -20,6 +21,14 @@ public class CrawlerStore extends BaseEntity {
private
String
type
;
private
String
shopAddr
;
private
String
address
;
private
String
email
;
private
JSONObject
extInfo
;
public
String
getExtInfo
()
{
if
(
extInfo
==
null
)
return
"{}"
;
return
extInfo
.
toString
();
}
}
src/main/java/com/deeprec/crawler/mapper/InsertDB.java
View file @
074bac95
...
...
@@ -3,18 +3,27 @@ package com.deeprec.crawler.mapper;
import
cn.hutool.db.Db
;
import
cn.hutool.db.Entity
;
import
com.deeprec.crawler.entry.CrawlerStore
;
import
lombok.SneakyThrows
;
import
lombok.extern.slf4j.Slf4j
;
import
java.sql.SQLException
;
@Slf4j
public
class
InsertDB
{
private
static
InsertDB
insertDB
;
private
InsertDB
()
{}
public
static
InsertDB
getDB
()
{
return
new
InsertDB
();
if
(
insertDB
==
null
)
{
insertDB
=
new
InsertDB
();
}
return
insertDB
;
}
public
boolean
insert
(
CrawlerStore
store
)
throws
SQLException
{
Entity
entity
=
Entity
.
create
(
"crawler_store"
).
parseBean
(
store
,
true
,
true
);
@SneakyThrows
public
boolean
insert
(
CrawlerStore
store
)
{
Entity
entity
=
Entity
.
create
(
"crawler_store"
)
.
parseBean
(
store
,
true
,
true
);
return
Db
.
use
().
insert
(
entity
)
==
0
;
}
}
src/main/java/com/deeprec/crawler/modules/eebbk/crawler/AuthorizeStationCrawler.java
0 → 100644
View file @
074bac95
package
com
.
deeprec
.
crawler
.
modules
.
eebbk
.
crawler
;
import
cn.hutool.http.HttpUtil
;
import
cn.hutool.json.JSONObject
;
import
com.deeprec.crawler.modules.eebbk.dto.AuthorizeStationParam
;
import
com.deeprec.crawler.modules.eebbk.dto.AuthorizeStationResult
;
import
com.deeprec.crawler.entry.CrawlerStore
;
import
com.deeprec.crawler.mapper.InsertDB
;
import
lombok.SneakyThrows
;
import
lombok.extern.slf4j.Slf4j
;
import
java.util.List
;
import
java.util.concurrent.TimeUnit
;
@Slf4j
public
class
AuthorizeStationCrawler
{
private
static
final
String
SK_NAME
=
"步步高"
;
private
static
final
String
URL
=
"https://www.eebbk.com/html/pc/service/service.html"
;
@SneakyThrows
public
static
void
main
(
String
[]
args
)
{
// 授权服务站点
String
station
=
HttpUtil
.
createGet
(
"https://service-center.okii.com/api/official-site/area"
)
.
form
(
"siteType"
,
"station"
)
.
execute
().
body
();
List
<
AuthorizeStationParam
>
directArray
=
new
JSONObject
(
station
)
.
getJSONArray
(
"data"
)
.
toList
(
AuthorizeStationParam
.
class
);
for
(
AuthorizeStationParam
param
:
directArray
)
{
String
request
=
HttpUtil
.
createGet
(
"https://service-center.okii.com/api/official-site/sites"
)
.
form
(
"pageIndex"
,
1
)
.
form
(
"pageSize"
,
1000
)
.
form
(
"siteType"
,
"station"
)
.
form
(
"officialProvince"
,
param
.
getProvince
())
.
form
(
"officialCity"
,
param
.
getCity
())
.
form
(
"officialCounty"
,
param
.
getCounty
())
.
execute
().
body
();
List
<
AuthorizeStationResult
>
authorizeStationResults
=
new
JSONObject
(
request
)
.
getJSONArray
(
"data"
)
.
toList
(
AuthorizeStationResult
.
class
);
for
(
AuthorizeStationResult
result
:
authorizeStationResults
)
{
CrawlerStore
crawlerStore
=
new
CrawlerStore
()
.
setSkName
(
SK_NAME
)
.
setCrawlerUrl
(
URL
)
.
setType
(
"授权服务站"
)
.
setName
(
result
.
getSiteName
())
.
setPhone
(
result
.
getTel
())
.
setAddress
(
result
.
getAddress
())
.
setExtInfo
(
new
JSONObject
()
.
set
(
"province"
,
result
.
getOfficialProvince
())
.
set
(
"restTime"
,
result
.
getRestTime
())
.
set
(
"city"
,
result
.
getOfficialCity
())
.
set
(
"authorizationCode"
,
result
.
getAuthorizationCode
())
.
set
(
"showNameBbk"
,
result
.
getShowNameBbk
())
.
set
(
"businessTime"
,
result
.
getBusinessTime
())
.
set
(
"county"
,
result
.
getOfficialCounty
())
.
set
(
"county"
,
result
.
getShowNameXtc
()));
log
.
info
(
"区域: {},{}"
,
result
.
getAddress
(),
crawlerStore
);
InsertDB
.
getDB
().
insert
(
crawlerStore
);
}
TimeUnit
.
MILLISECONDS
.
sleep
(
300
);
}
log
.
info
(
"授权服务站点全部插入完成"
);
}
}
src/main/java/com/deeprec/crawler/modules/eebbk/crawler/CustomerServiceCrawler.java
0 → 100644
View file @
074bac95
package
com
.
deeprec
.
crawler
.
modules
.
eebbk
.
crawler
;
import
cn.hutool.http.HttpUtil
;
import
cn.hutool.json.JSONObject
;
import
com.deeprec.crawler.modules.eebbk.dto.CustomerServiceParam
;
import
com.deeprec.crawler.modules.eebbk.dto.CustomerServiceResult
;
import
com.deeprec.crawler.entry.CrawlerStore
;
import
com.deeprec.crawler.mapper.InsertDB
;
import
lombok.SneakyThrows
;
import
lombok.extern.slf4j.Slf4j
;
import
java.util.List
;
import
java.util.concurrent.TimeUnit
;
@Slf4j
public
class
CustomerServiceCrawler
{
private
static
final
String
SK_NAME
=
"步步高"
;
private
static
final
String
URL
=
"https://www.eebbk.com/html/pc/service/service.html"
;
@SneakyThrows
public
static
void
main
(
String
[]
args
)
{
String
direct
=
HttpUtil
.
createGet
(
"https://service-center.okii.com/api/official-site/area"
)
.
form
(
"siteType"
,
"direct"
)
.
execute
().
body
();
List
<
CustomerServiceParam
>
directArray
=
new
JSONObject
(
direct
).
getJSONArray
(
"data"
)
.
toList
(
CustomerServiceParam
.
class
);
for
(
CustomerServiceParam
param
:
directArray
)
{
String
request
=
HttpUtil
.
createGet
(
"https://service-center.okii.com/api/official-site/sites"
)
.
form
(
"pageIndex"
,
1
)
.
form
(
"pageSize"
,
1000
)
.
form
(
"siteType"
,
"direct"
)
.
form
(
"officialProvince"
,
param
.
getProvince
())
.
form
(
"officialCity"
,
param
.
getCity
())
.
execute
().
body
();
List
<
CustomerServiceResult
>
salesOutlets
=
new
JSONObject
(
request
)
.
getJSONArray
(
"data"
)
.
toList
(
CustomerServiceResult
.
class
);
for
(
CustomerServiceResult
result
:
salesOutlets
)
{
CrawlerStore
crawlerStore
=
new
CrawlerStore
()
.
setSkName
(
SK_NAME
)
.
setCrawlerUrl
(
URL
)
.
setType
(
"客服中心"
)
.
setName
(
result
.
getSiteName
())
.
setPhone
(
result
.
getTel
())
.
setAddress
(
result
.
getAddress
())
.
setExtInfo
(
new
JSONObject
()
.
set
(
"province"
,
result
.
getOfficialProvince
())
.
set
(
"city"
,
result
.
getOfficialCity
())
.
set
(
"county"
,
result
.
getOfficialCounty
())
.
set
(
"openingHours"
,
result
.
getBusinessTime
())
.
set
(
"restTime"
,
result
.
getRestTime
())
.
set
(
"showNameXtc"
,
result
.
getShowNameXtc
())
.
set
(
"showNameBbk"
,
result
.
getShowNameBbk
()));
log
.
info
(
"区域: {},{}"
,
result
.
getAddress
(),
crawlerStore
);
InsertDB
.
getDB
().
insert
(
crawlerStore
);
}
TimeUnit
.
MILLISECONDS
.
sleep
(
300
);
}
log
.
info
(
"客服中心全部插入完成"
);
}
}
src/main/java/com/deeprec/crawler/modules/eebbk/crawler/SalesOutlesEebbkCrawler.java
0 → 100644
View file @
074bac95
package
com
.
deeprec
.
crawler
.
modules
.
eebbk
.
crawler
;
import
cn.hutool.http.HttpUtil
;
import
cn.hutool.json.JSONObject
;
import
com.deeprec.crawler.modules.eebbk.dto.SalesOutletsParam
;
import
com.deeprec.crawler.modules.eebbk.dto.SalesOutletsResult
;
import
com.deeprec.crawler.entry.CrawlerStore
;
import
com.deeprec.crawler.mapper.InsertDB
;
import
lombok.SneakyThrows
;
import
lombok.extern.slf4j.Slf4j
;
import
java.util.List
;
import
java.util.concurrent.TimeUnit
;
@Slf4j
public
class
SalesOutlesEebbkCrawler
{
private
static
final
String
SK_NAME
=
"步步高"
;
private
static
final
String
URL
=
"https://www.eebbk.com/html/pc/service/service.html"
;
@SneakyThrows
public
static
void
main
(
String
[]
args
)
{
String
salesOutletsParam
=
HttpUtil
.
createGet
(
"https://scrmapis.okii.com/area"
).
execute
().
body
();
List
<
SalesOutletsParam
>
areas
=
new
JSONObject
(
salesOutletsParam
)
.
getJSONArray
(
"data"
)
.
toList
(
SalesOutletsParam
.
class
);
for
(
SalesOutletsParam
area
:
areas
)
{
String
request
=
HttpUtil
.
createGet
(
"https://www.eebbk.com/api/network/sales"
)
.
form
(
"brand"
,
"eebbk"
)
.
form
(
"pageIndex"
,
1
)
.
form
(
"province"
,
area
.
getProvince
())
.
form
(
"city"
,
area
.
getCity
())
.
form
(
"county"
,
area
.
getCounty
())
.
execute
().
body
();
List
<
SalesOutletsResult
>
salesOutlets
=
new
JSONObject
(
request
)
.
getJSONArray
(
"data"
)
.
toList
(
SalesOutletsResult
.
class
);
for
(
SalesOutletsResult
e
:
salesOutlets
)
{
CrawlerStore
crawlerStore
=
new
CrawlerStore
()
.
setSkName
(
SK_NAME
)
.
setCrawlerUrl
(
URL
)
.
setName
(
e
.
getTitle
())
.
setPhone
(
e
.
getTel
())
.
setAddress
(
e
.
getAddress
())
.
setType
(
"销售网点"
)
.
setExtInfo
(
new
JSONObject
()
.
set
(
"province"
,
e
.
getProvince
())
.
set
(
"city"
,
e
.
getCity
())
.
set
(
"county"
,
e
.
getCounty
())
.
set
(
"openingHours"
,
e
.
getOpeningHours
()));
log
.
info
(
"区域: {},{}"
,
e
.
getAddress
(),
crawlerStore
);
InsertDB
.
getDB
().
insert
(
crawlerStore
);
}
TimeUnit
.
MILLISECONDS
.
sleep
(
300
);
}
log
.
info
(
"销售网点全部插入完成"
);
}
}
src/main/java/com/deeprec/crawler/modules/eebbk/dto/AuthorizeStationParam.java
0 → 100644
View file @
074bac95
package
com
.
deeprec
.
crawler
.
modules
.
eebbk
.
dto
;
import
lombok.Data
;
// 步步高 授权站点查询结构
@Data
public
class
AuthorizeStationParam
{
String
province
;
String
city
;
String
county
;
}
src/main/java/com/deeprec/crawler/modules/eebbk/dto/AuthorizeStationResult.java
0 → 100644
View file @
074bac95
package
com
.
deeprec
.
crawler
.
modules
.
eebbk
.
dto
;
import
lombok.Data
;
@Data
public
class
AuthorizeStationResult
{
String
officialProvince
;
String
restTime
;
String
address
;
String
officialCity
;
String
authorizationCode
;
String
showNameBbk
;
String
siteName
;
String
businessTime
;
String
officialCounty
;
String
tel
;
String
showNameXtc
;
}
src/main/java/com/deeprec/crawler/modules/eebbk/dto/CustomerServiceParam.java
0 → 100644
View file @
074bac95
package
com
.
deeprec
.
crawler
.
modules
.
eebbk
.
dto
;
import
lombok.Data
;
// 步步高 客服中心查询结构
@Data
public
class
CustomerServiceParam
{
String
province
;
String
city
;
}
src/main/java/com/deeprec/crawler/modules/eebbk/dto/CustomerServiceResult.java
0 → 100644
View file @
074bac95
package
com
.
deeprec
.
crawler
.
modules
.
eebbk
.
dto
;
import
lombok.Data
;
@Data
public
class
CustomerServiceResult
{
String
siteName
;
String
tel
;
String
address
;
String
officialProvince
;
String
officialCity
;
String
officialCounty
;
String
wechatWorkId
;
String
businessTime
;
String
restTime
;
String
notice
;
String
showNameXtc
;
String
showNameBbk
;
}
src/main/java/com/deeprec/crawler/modules/eebbk/dto/SalesOutletsParam.java
0 → 100644
View file @
074bac95
package
com
.
deeprec
.
crawler
.
modules
.
eebbk
.
dto
;
import
lombok.Data
;
// 步步高 销售网点区域查询结构
@Data
public
class
SalesOutletsParam
{
String
province
;
String
city
;
String
county
;
}
src/main/java/com/deeprec/crawler/modules/eebbk/dto/SalesOutletsResult.java
0 → 100644
View file @
074bac95
package
com
.
deeprec
.
crawler
.
modules
.
eebbk
.
dto
;
import
lombok.Data
;
// 步步高 销售网点区域结果返回
@Data
public
class
SalesOutletsResult
{
String
title
;
String
province
;
String
city
;
String
county
;
String
address
;
String
tel
;
String
openingHours
;
}
src/main/java/com/deeprec/crawler/modules/familyMart/crawler/FamilyMartCrawler.java
0 → 100644
View file @
074bac95
package
com
.
deeprec
.
crawler
.
modules
.
familyMart
.
crawler
;
import
cn.hutool.http.HttpUtil
;
import
cn.hutool.json.JSONObject
;
import
com.deeprec.crawler.entry.CrawlerStore
;
import
com.deeprec.crawler.mapper.InsertDB
;
import
com.deeprec.crawler.modules.familyMart.dto.Store
;
import
lombok.extern.slf4j.Slf4j
;
import
org.jsoup.Jsoup
;
import
org.jsoup.nodes.Document
;
import
org.jsoup.nodes.Element
;
import
org.jsoup.select.Elements
;
import
java.util.LinkedHashMap
;
import
java.util.List
;
import
java.util.Map
;
@Slf4j
public
class
FamilyMartCrawler
{
private
static
final
String
SK_NAME
=
"全家"
;
private
static
final
String
URL
=
"http://www.familymart.com.cn/store"
;
private
static
final
Map
<
String
,
String
>
citys
=
new
LinkedHashMap
<>();
/**
* /store/Search?cid=1&page=1 POST请求
* cid写死在页面上 上海、苏州、深圳、广州、杭州、成都 分别是1至6
* 无锡、北京、东莞、嘉兴为10至13
* 返回的结果为
* {
* "city": "上海",
* "mapmsg": [,...], --> 所有结果一共3045条数据
* "msg": [,...], --> 当前分页的10条数据
* "nowpage": "1",
* "preferen": false,
* "totalcount": "3045",
* "totalpage": 305
* }
* 直接使用返回的mapmsg即可
*/
public
static
void
main
(
String
[]
args
)
{
String
response
=
HttpUtil
.
createPost
(
"http://www.familymart.com.cn/store"
).
execute
().
body
();
Document
document
=
Jsoup
.
parse
(
response
);
Elements
elements
=
document
.
getElementsByClass
(
"searchbox-sc"
);
String
html
=
elements
.
html
();
Elements
options
=
Jsoup
.
parse
(
html
).
getElementsByTag
(
"option"
);
for
(
Element
option
:
options
)
{
citys
.
put
(
option
.
text
(),
option
.
val
());
}
citys
.
forEach
((
cityName
,
cid
)
->
{
String
station
=
HttpUtil
.
createPost
(
"http://www.familymart.com.cn/store/Search"
)
.
form
(
"cid"
,
cid
)
.
form
(
"aid"
)
.
form
(
"street"
)
.
form
(
"address"
)
.
form
(
"service"
)
.
form
(
"name"
)
.
form
(
"page"
,
1
)
.
execute
().
body
();
List
<
Store
>
stores
=
new
JSONObject
(
station
).
getJSONArray
(
"mapmsg"
).
toList
(
Store
.
class
);
stores
.
forEach
(
e
->
{
CrawlerStore
crawlerStore
=
new
CrawlerStore
().
setCrawlerUrl
(
URL
)
.
setSkName
(
SK_NAME
).
setPhone
(
e
.
getTelphone
())
.
setType
(
cityName
)
.
setName
(
e
.
getName
())
.
setAddress
(
e
.
getStreet
())
.
setExtInfo
(
new
JSONObject
().
set
(
"createtime"
,
e
.
getCreatetime
())
.
set
(
"ts"
,
e
.
getTs
())
);
InsertDB
.
getDB
().
insert
(
crawlerStore
);
log
.
info
(
"区域: {},{}"
,
crawlerStore
.
getAddress
(),
crawlerStore
);
});
});
log
.
info
(
"全家便利店全部插入完成"
);
}
}
src/main/java/com/deeprec/crawler/modules/familyMart/dto/Store.java
0 → 100644
View file @
074bac95
package
com
.
deeprec
.
crawler
.
modules
.
familyMart
.
dto
;
import
lombok.Data
;
@Data
public
class
Store
{
String
createtime
;
String
uname
;
String
last_sync_time
;
String
zm
;
String
street
;
String
service
;
String
telphone
;
String
name
;
String
cid
;
String
ts
;
}
src/main/java/com/deeprec/crawler/modules/maidelong/crawler/MaidelongCrawler.java
0 → 100644
View file @
074bac95
package
com
.
deeprec
.
crawler
.
modules
.
maidelong
.
crawler
;
import
cn.hutool.http.HttpUtil
;
import
cn.hutool.json.JSONArray
;
import
cn.hutool.json.JSONObject
;
import
com.deeprec.crawler.entry.CrawlerStore
;
import
com.deeprec.crawler.modules.maidelong.dto.Store
;
import
com.deeprec.crawler.mapper.InsertDB
;
import
lombok.extern.slf4j.Slf4j
;
import
java.util.List
;
@Slf4j
public
class
MaidelongCrawler
{
private
static
final
String
SK_NAME
=
"麦德龙"
;
private
static
final
String
URL
=
"https://www.maidelong.com/newmetro-frontend-web/#/stores/165"
;
public
static
void
main
(
String
[]
args
)
{
String
response
=
HttpUtil
.
createGet
(
"https://www.maidelong.com/newmetro-frontend-api/stores/getStoresInfo"
)
.
form
(
"lang"
,
"zh-cn"
)
.
form
(
"curLat"
,
""
)
.
form
(
"curLng"
,
""
)
.
form
(
"maxDistance"
,
"100000000"
)
.
form
(
"limit"
,
"999"
)
.
execute
().
body
();
JSONArray
jsonArray
=
new
JSONObject
(
response
).
getJSONObject
(
"data"
).
getJSONArray
(
"storesData"
);
List
<
Store
>
stores
=
jsonArray
.
toList
(
Store
.
class
);
stores
.
forEach
(
e
->
{
CrawlerStore
crawlerStore
=
new
CrawlerStore
()
.
setCrawlerUrl
(
URL
)
.
setSkName
(
SK_NAME
)
.
setName
(
e
.
getName
())
.
setType
(
e
.
getRegionName
())
.
setEmail
(
e
.
getEmail
())
.
setAddress
(
e
.
getAddress
())
.
setPhone
(
e
.
getTelephone
())
.
setExtInfo
(
new
JSONObject
().
set
(
"manger"
,
e
.
getManagerCN
())
.
set
(
"openingHours"
,
e
.
getOpeningHoursCN
())
.
set
(
"postCode"
,
e
.
getPostcode
()));
InsertDB
.
getDB
().
insert
(
crawlerStore
);
log
.
info
(
e
.
getName
()
+
" - "
+
e
.
getRegionName
()
+
" -> 导入"
);
});
log
.
info
(
SK_NAME
+
"全部导入完成!"
);
}
}
src/main/java/com/deeprec/crawler/modules/maidelong/dto/Store.java
0 → 100644
View file @
074bac95
package
com
.
deeprec
.
crawler
.
modules
.
maidelong
.
dto
;
import
lombok.Data
;
@Data
public
class
Store
{
String
regionName
;
String
openingHoursCN
;
String
managerCN
;
String
email
;
String
address
;
String
postcode
;
String
telephone
;
String
name
;
}
src/main/java/com/deeprec/crawler/suguo/SuguoCrawler.java
→
src/main/java/com/deeprec/crawler/
modules/
suguo/SuguoCrawler.java
View file @
074bac95
package
com
.
deeprec
.
crawler
.
suguo
;
package
com
.
deeprec
.
crawler
.
modules
.
suguo
;
import
cn.hutool.http.HttpUtil
;
import
com.deeprec.crawler.entry.CrawlerStore
;
...
...
@@ -61,7 +61,7 @@ public class SuguoCrawler {
.
setSkName
(
SK_NAME
)
.
setName
(
shopName
)
.
setType
(
shopType
)
.
set
ShopAddr
(
shopAddr
)
.
set
Address
(
shopAddr
)
.
setPhone
(
shopPhone
);
InsertDB
.
getDB
().
insert
(
crawlerStore
);
}
...
...
src/main/java/com/deeprec/crawler/modules/yonghui/YonghuiCrawler.java
0 → 100644
View file @
074bac95
package
com
.
deeprec
.
crawler
.
modules
.
yonghui
;
import
cn.hutool.http.HttpUtil
;
import
cn.hutool.json.JSONObject
;
import
com.deeprec.crawler.entry.CrawlerStore
;
import
com.deeprec.crawler.mapper.InsertDB
;
import
lombok.extern.slf4j.Slf4j
;
import
org.jsoup.Jsoup
;
import
org.jsoup.nodes.Document
;
import
org.jsoup.nodes.Element
;
import
org.jsoup.select.Elements
;
@Slf4j
public
class
YonghuiCrawler
{
private
static
final
String
SK_NAME
=
"永辉超市"
;
private
static
final
String
URL
=
"https://www.yonghui.com.cn/store_list"
;
public
static
void
main
(
String
[]
args
)
{
int
page
=
0
;
while
(
true
)
{
String
station
=
HttpUtil
.
createGet
(
"https://www.yonghui.com.cn/mapi/proajax"
)
.
form
(
"act"
,
2
)
.
form
(
"pid"
,
0
)
.
form
(
"ctlgid"
,
0
)
.
form
(
"keyword"
)
.
form
(
"p"
,
page
++)
.
form
(
"rnd"
,
"742.4132773474001"
)
.
execute
().
body
();
String
isComplete
=
station
.
substring
(
5
);
if
(
"暂无数据"
.
equals
(
isComplete
))
{
break
;
}
Document
document
=
Jsoup
.
parse
(
station
);
Elements
elements
=
document
.
getElementsByClass
(
"storeList"
);
for
(
Element
element
:
elements
)
{
Elements
lis
=
element
.
getElementsByTag
(
"li"
);
for
(
Element
li
:
lis
)
{
String
name
=
li
.
getElementsByTag
(
"h1"
).
text
();
Elements
span
=
li
.
getElementsByTag
(
"span"
);
String
[]
split
=
span
.
html
().
split
(
"<br>"
);
split
[
split
.
length
-
1
]
=
""
;
String
address
=
split
[
0
].
substring
(
5
);
String
phone
=
split
[
1
].
substring
(
5
);
String
openingTime
=
split
[
2
].
substring
(
5
);
String
busRoute
=
split
[
3
].
substring
(
5
)
+
split
[
4
];
CrawlerStore
crawlerStore
=
new
CrawlerStore
()
.
setSkName
(
SK_NAME
)
.
setCrawlerUrl
(
URL
)
.
setType
(
"已开业"
)
.
setName
(
name
)
.
setPhone
(
phone
)
.
setAddress
(
address
)
.
setExtInfo
(
new
JSONObject
()
.
set
(
"openingTime"
,
openingTime
)
.
set
(
"busRoute"
,
busRoute
));
InsertDB
.
getDB
().
insert
(
crawlerStore
);
log
.
info
(
name
+
" - "
+
address
+
" -> 导入"
);
}
}
}
log
.
info
(
SK_NAME
+
"已开业店铺导入完成!"
);
}
}
src/main/java/com/deeprec/crawler/modules/yonghui/YonghuiOfPrepareCrawler.java
0 → 100644
View file @
074bac95
package
com
.
deeprec
.
crawler
.
modules
.
yonghui
;
import
cn.hutool.http.HttpUtil
;
import
cn.hutool.json.JSONObject
;
import
com.deeprec.crawler.entry.CrawlerStore
;
import
com.deeprec.crawler.mapper.InsertDB
;
import
lombok.extern.slf4j.Slf4j
;
import
org.jsoup.Jsoup
;
import
org.jsoup.nodes.Document
;
import
org.jsoup.nodes.Element
;
import
org.jsoup.select.Elements
;
@Slf4j
public
class
YonghuiOfPrepareCrawler
{
private
static
final
String
SK_NAME
=
"永辉超市"
;
private
static
final
String
URL
=
"https://www.yonghui.com.cn/store_list"
;
public
static
void
main
(
String
[]
args
)
{
int
page
=
0
;
while
(
true
)
{
String
station
=
HttpUtil
.
createGet
(
"https://www.yonghui.com.cn/mapi/proajax"
)
.
form
(
"act"
,
3
)
.
form
(
"pid"
,
0
)
.
form
(
"ctlgid"
,
0
)
.
form
(
"keyword"
)
.
form
(
"p"
,
page
++)
.
form
(
"rnd"
,
"742.4132773474001"
)
.
execute
().
body
();
String
isComplete
=
station
.
substring
(
4
);
if
(
"暂无数据"
.
equals
(
isComplete
))
{
break
;
}
Document
document
=
Jsoup
.
parse
(
station
);
Elements
elements
=
document
.
getElementsByClass
(
"storeList"
);
for
(
Element
element
:
elements
)
{
Elements
lis
=
element
.
getElementsByTag
(
"li"
);
for
(
Element
li
:
lis
)
{
String
name
=
li
.
getElementsByTag
(
"h1"
).
text
();
Elements
span
=
li
.
getElementsByTag
(
"span"
);
String
[]
split
=
span
.
html
().
split
(
"<br>"
);
String
address
=
split
[
0
].
substring
(
5
);
CrawlerStore
crawlerStore
=
new
CrawlerStore
()
.
setSkName
(
SK_NAME
)
.
setCrawlerUrl
(
URL
)
.
setType
(
"筹备中"
)
.
setName
(
name
)
.
setExtInfo
(
new
JSONObject
().
set
(
"openingTime"
,
split
[
1
].
substring
(
5
)));
InsertDB
.
getDB
().
insert
(
crawlerStore
);
log
.
info
(
name
+
" - "
+
address
+
" -> 导入"
);
}
}
}
log
.
info
(
SK_NAME
+
"已开业店铺导入完成!"
);
}
}
src/main/java/com/deeprec/crawler/utils/AspectUtil.java
0 → 100644
View file @
074bac95
package
com
.
deeprec
.
crawler
.
utils
;
import
cn.hutool.aop.aspects.SimpleAspect
;
import
java.lang.reflect.Method
;
import
java.util.List
;
public
class
AspectUtil
extends
SimpleAspect
{
private
List
<
Object
>
list
;
@Override
public
boolean
before
(
Object
target
,
Method
method
,
Object
[]
args
)
{
return
true
;
}
@Override
public
boolean
after
(
Object
target
,
Method
method
,
Object
[]
args
,
Object
returnVal
)
{
return
true
;
}
@Override
public
boolean
afterException
(
Object
target
,
Method
method
,
Object
[]
args
,
Throwable
e
)
{
return
true
;
}
}
src/main/java/com/deeprec/crawler/utils/CrawlerUtil.java
View file @
074bac95
...
...
@@ -18,14 +18,13 @@ public class CrawlerUtil {
if
(
param
!=
null
)
bindings
.
putAll
(
param
);
Object
result
=
JSFunction
.
eval
(
bindings
);
ScriptObjectMirror
scpt
=
(
ScriptObjectMirror
)
result
;
ScriptObjectMirror
scpt
=
(
ScriptObjectMirror
)
JSFunction
.
eval
(
bindings
);
List
<
Object
>
list
=
new
ArrayList
<>(
scpt
.
values
());
List
<
String
>
r
=
new
ArrayList
<>();
for
(
Object
o
:
list
)
{
r
.
add
(
o
.
toString
());
}
return
r
;
return
r
;
}
}
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment