init

.gitignore

+/.idea/
+/.idea/*
+/classes/
+build
+out
+*.DS_Store
+logs
+*.iml
+*./class
+.idea/libraries/*
+
+
+# Created by .ignore support plugin (hsz.mobi)
+### Gradle template
+.gradle
+/build/
+
+# Ignore Gradle GUI config
+gradle-app.setting
+
+# Avoid ignoring Gradle wrapper jar file (.jar files are usually ignored)
+!gradle-wrapper.jar
+
+# Cache of project
+.gradletasknamecache
+
+# # Work around https://youtrack.jetbrains.com/issue/IDEA-116898
+# gradle/wrapper/gradle-wrapper.properties
+### NetBeans template
+nbproject/private/
+build/
+nbbuild/
+dist/
+nbdist/
+.nb-gradle/
+### Eclipse template
+
+.metadata
+bin/
+tmp/
+*.tmp
+*.bak
+*.swp
+*~.nib
+local.properties
+.settings/
+.loadpath
+.recommenders
+
+# External tool builders
+.externalToolBuilders/
+
+# Locally stored "Eclipse launch configurations"
+*.launch
+
+# PyDev specific (Python IDE for Eclipse)
+*.pydevproject
+
+# CDT-specific (C/C++ Development Tooling)
+.cproject
+
+# Java annotation processor (APT)
+.factorypath
+
+# PDT-specific (PHP Development Tools)
+.buildpath
+
+# sbteclipse plugin
+.target
+
+# Tern plugin
+.tern-project
+
+# TeXlipse plugin
+.texlipse
+
+# STS (Spring Tool Suite)
+.springBeans
+
+# Code Recommenders
+.recommenders/
+
+# Scala IDE specific (Scala & Java development for Eclipse)
+.cache-main
+.scala_dependencies
+.worksheet
+### C++ template
+# Prerequisites
+*.d
+
+# Compiled Object files
+*.slo
+*.lo
+*.o
+*.obj
+
+# Precompiled Headers
+*.gch
+*.pch
+
+# Compiled Dynamic libraries
+*.so
+*.dylib
+*.dll
+
+# Fortran module files
+*.mod
+*.smod
+
+# Compiled Static libraries
+*.lai
+*.la
+*.a
+*.lib
+
+# Executables
+*.exe
+*.out
+*.app
+### C template
+# Prerequisites
+*.d
+
+# Object files
+*.o
+*.ko
+*.obj
+*.elf
+
+# Linker output
+*.ilk
+*.map
+*.exp
+
+# Precompiled Headers
+*.gch
+*.pch
+
+# Libraries
+*.lib
+*.a
+*.la
+*.lo
+
+# Shared objects (inc. Windows DLLs)
+*.dll
+*.so
+*.so.*
+*.dylib
+
+# Executables
+*.exe
+*.out
+*.app
+*.i*86
+*.x86_64
+*.hex
+
+# Debug files
+*.dSYM/
+*.su
+*.idb
+*.pdb
+
+# Kernel Module Compile Results
+*.mod*
+*.cmd
+.tmp_versions/
+modules.order
+Module.symvers
+Mkfile.old
+dkms.conf
+### Java template
+# Compiled class file
+*.class
+
+# Log file
+*.log
+
+# BlueJ files
+*.ctxt
+
+# Mobile Tools for Java (J2ME)
+.mtj.tmp/
+
+# Package Files #
+*.jar
+*.war
+*.ear
+*.zip
+*.tar.gz
+*.rar
+
+# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
+hs_err_pid*
+### Maven template
+target/
+pom.xml.tag
+pom.xml.releaseBackup
+pom.xml.versionsBackup
+pom.xml.next
+release.properties
+dependency-reduced-pom.xml
+buildNumber.properties
+.mvn/timing.properties
+
+# Avoid ignoring Maven wrapper jar file (.jar files are usually ignored)
+!/.mvn/wrapper/maven-wrapper.jar
+### JetBrains template
+# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm
+# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
+
+# User-specific stuff:
+.idea/**/workspace.xml
+.idea/**/tasks.xml
+.idea/dictionaries
+
+# Sensitive or high-churn files:
+.idea/**/dataSources/
+.idea/**/dataSources.ids
+.idea/**/dataSources.xml
+.idea/**/dataSources.local.xml
+.idea/**/sqlDataSources.xml
+.idea/**/dynamic.xml
+.idea/**/uiDesigner.xml
+
+# Gradle:
+.idea/**/gradle.xml
+.idea/**/libraries
+
+# CMake
+cmake-build-debug/
+
+# Mongo Explorer plugin:
+.idea/**/mongoSettings.xml
+
+## File-based project format:
+*.iws
+
+## Plugin-specific files:
+
+# IntelliJ
+out/
+
+# mpeltonen/sbt-idea plugin
+.idea_modules/
+
+# JIRA plugin
+atlassian-ide-plugin.xml
+
+# Cursive Clojure plugin
+.idea/replstate.xml
+
+# Crashlytics plugin (for Android Studio and IntelliJ)
+com_crashlytics_export_strings.xml
+crashlytics.properties
+crashlytics-build.properties
+fabric.properties
+
+### Example user template template
+### Example user template
+
+# IntelliJ project files
+.idea
+*.iml
+out
+gen
+
+# Eclipse
+.project
+.classpath
+.settings
+.DS_Store
\ No newline at end of file

pom.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>com.deeprec</groupId>
+    <artifactId>Crawler</artifactId>
+    <version>1.0-SNAPSHOT</version>
+
+    <properties>
+        <maven.compiler.source>8</maven.compiler.source>
+        <maven.compiler.target>8</maven.compiler.target>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>cn.hutool</groupId>
+            <artifactId>hutool-all</artifactId>
+            <version>5.7.21</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.jsoup</groupId>
+            <artifactId>jsoup</artifactId>
+            <version>1.14.3</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.logging.log4j</groupId>
+            <artifactId>log4j-slf4j-impl</artifactId>
+            <version>2.11.0</version>
+        </dependency>
+        <dependency>
+            <groupId>org.projectlombok</groupId>
+            <artifactId>lombok</artifactId>
+            <version>1.18.22</version>
+        </dependency>
+
+        <dependency>
+            <groupId>mysql</groupId>
+            <artifactId>mysql-connector-java</artifactId>
+            <version>8.0.21</version>
+        </dependency>
+
+    </dependencies>
+</project>
\ No newline at end of file

src/main/java/com/deeprec/crawler/entry/BaseEntity.java

+package com.deeprec.crawler.entry;
+
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.experimental.Accessors;
+
+import java.io.Serializable;
+import java.util.Date;
+
+@Data
+@EqualsAndHashCode(callSuper = false)
+@Accessors(chain = true)
+public class BaseEntity implements Serializable {
+
+    private Long id;
+
+    private Long createBy = 2L ;
+
+    private Date createTime = new Date();
+
+    private Long updateBy;
+
+    private Date updateTime;
+
+    private Integer delFlag = 0;
+
+}

src/main/java/com/deeprec/crawler/entry/CrawlerStore.java

+package com.deeprec.crawler.entry;
+
+import lombok.Data;
+
+import lombok.EqualsAndHashCode;
+import lombok.experimental.Accessors;
+
+@Data
+@EqualsAndHashCode(callSuper = false)
+@Accessors(chain = true)
+public class CrawlerStore extends BaseEntity {
+
+    private String name;
+
+    private String skName;
+
+    private String crawlerUrl;
+
+    private String phone;
+
+    private String type;
+
+    private String shopAddr;
+
+}

src/main/java/com/deeprec/crawler/mapper/InsertDB.java

+package com.deeprec.crawler.mapper;
+
+import cn.hutool.db.Db;
+import cn.hutool.db.Entity;
+import com.deeprec.crawler.entry.CrawlerStore;
+
+import java.sql.SQLException;
+
+public class InsertDB {
+
+    public static InsertDB getDB() {
+        return new InsertDB();
+    }
+
+    public boolean insert(CrawlerStore store) throws SQLException {
+        Entity entity = Entity.create("crawler_store").parseBean(store, true, true);
+        return Db.use().insert(entity) == 0;
+    }
+
+}

src/main/java/com/deeprec/crawler/suguo/SuguoCrawler.java

+package com.deeprec.crawler.suguo;
+
+import cn.hutool.http.HttpUtil;
+import com.deeprec.crawler.entry.CrawlerStore;
+import com.deeprec.crawler.mapper.InsertDB;
+import com.deeprec.crawler.utils.CrawlerUtil;
+import lombok.extern.slf4j.Slf4j;
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+import org.jsoup.select.Elements;
+
+import javax.script.ScriptException;
+import java.io.UnsupportedEncodingException;
+import java.net.URLEncoder;
+import java.sql.SQLException;
+import java.util.List;
+import java.util.stream.Collectors;
+
+@Slf4j
+public class SuguoCrawler {
+    private static final String SK_NAME = "苏果";
+    private static final String URL = "https://suguovip.suguo.com.cn/Web/shopResult.m";
+
+    public static void main(String[] args) throws UnsupportedEncodingException, ScriptException, SQLException {
+        List<String> cityNames = init();
+
+        for (String cityName : cityNames) {
+            int currentPage = 1;
+            String encodeCity = URLEncoder.encode(URLEncoder.encode(cityName, "utf-8"), "utf-8");
+
+            // shopType 0 为全部
+            int totalPage = page(encodeCity);
+
+            while (totalPage >= currentPage) {
+                String url = URL + "?shopType=" + 0 + "&cityName=" + encodeCity + "&maod&page=" + currentPage++;
+                String response = HttpUtil.createGet(url).execute().body();
+                Document document = Jsoup.parse(response);
+
+                // shopType 1 为大卖场 2为社区店 3为便利店
+                Elements list = document.getElementsByClass("content-detail");
+                for (Element element : list) {
+                    String shopType = null;
+                    if (element.parent() != null && element.parent().hasClass("hide")) continue;
+                    if (element.hasClass("store-type-1")) {
+                        shopType = "大卖场";
+                    }
+                    if (element.hasClass("store-type-2")) {
+                        shopType = "社区店";
+                    }
+                    if (element.hasClass("store-type-3")) {
+                        shopType = "便利店";
+                    }
+                    String shopName = element.getElementsByClass("shop-name").text();
+                    Elements elements = element.getElementsByClass("shop-detail");
+                    String shopAddr = elements.get(0).getElementsByClass("shop-addr").text().substring(5);
+                    String shopPhone = elements.get(1).getElementsByClass("shop-phone").text().substring(5);
+                    log.info(shopName + " - " + shopType + " - " + shopAddr + " - " + shopPhone + " -> 导入");
+                    CrawlerStore crawlerStore = new CrawlerStore()
+                            .setCrawlerUrl(URL)
+                            .setSkName(SK_NAME)
+                            .setName(shopName)
+                            .setType(shopType)
+                            .setShopAddr(shopAddr)
+                            .setPhone(shopPhone);
+                    InsertDB.getDB().insert(crawlerStore);
+                }
+            }
+            log.info("{} {}门店, 正在导入中...", SK_NAME, cityName);
+        }
+        log.info(SK_NAME + "全部导入完成！");
+    }
+
+    private static List<String> init() throws ScriptException {
+        String response = HttpUtil.createGet(URL).execute().body();
+        Document document = Jsoup.parse(response);
+        Elements elements = document.getElementsByTag("script");
+
+        String jsCode = elements.get(8).data();
+        return CrawlerUtil.executeScriptGetList(jsCode, null).stream()
+                .map(e -> e.split("\\|")[0])
+                .collect(Collectors.toList());
+    }
+
+    public static int page(String encodeCity) {
+        String url = URL + "?shopType=" + 0 + "&cityName=" + encodeCity + "&maod&page=" + 599;
+        String response = HttpUtil.createGet(url).execute().body();
+        Document document = Jsoup.parse(response);
+        Elements elements = document.getElementsByTag("script");
+        String jsCode = elements.get(11).data();
+        return Integer.parseInt(jsCode.split("pagecount: ")[1].split(",")[0]);
+    }
+}

src/main/java/com/deeprec/crawler/utils/CrawlerUtil.java

+package com.deeprec.crawler.utils;
+
+import jdk.nashorn.api.scripting.ScriptObjectMirror;
+
+import javax.script.*;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public class CrawlerUtil {
+
+    public static List<String> executeScriptGetList(String code, Map<String, String> param) throws ScriptException {
+        ScriptEngine engine = new ScriptEngineManager().getEngineByName("JavaScript");
+        Compilable compilable = (Compilable) engine;
+        Bindings bindings = engine.createBindings();
+        CompiledScript JSFunction = compilable.compile(code);
+
+        if (param != null)
+            bindings.putAll(param);
+
+        Object result = JSFunction.eval(bindings);
+        ScriptObjectMirror scpt = (ScriptObjectMirror) result;
+        List<Object> list = new ArrayList<>(scpt.values());
+
+        List<String> r = new ArrayList<>();
+        for (Object o : list) {
+            r.add(o.toString());
+        }
+        return r ;
+    }
+}

src/main/resources/db.setting

+url = jdbc:mysql://139.129.228.167:3306/lcd_server?characterEncoding=UTF-8&useUnicode=true&useSSL=false
+user = root
+pass = root
+
+showSql = true
+formatSql = false
+showParams = true
+sqlLevel = info
\ No newline at end of file

src/main/resources/log4j2.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<configuration status="INFO">
+    <appenders>
+        <Console name="Console" target="SYSTEM_OUT">
+            <PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"/>
+        </Console>
+        <RollingFile name="RollingFile" fileName="logs/app.log"
+                     filePattern="logs/$${date:yyyy-MM}/app-%d{MM-dd-yyyy}-%i.log.gz">
+            <PatternLayout pattern="%d{yyyy.MM.dd 'at' HH:mm:ss z} %-5level %class{36} %L %M - %msg%xEx%n"/>
+            <SizeBasedTriggeringPolicy size="5 MB"/>
+        </RollingFile>
+    </appenders>
+    <loggers>
+        <root level="DEBUG">
+            <appender-ref ref="Console"/>
+            <appender-ref ref="RollingFile"/>
+        </root>
+    </loggers>
+</configuration>