zhc077
2 years ago
parent
e3f6118c44
commit
2dbc5500d3
2 changed files with 75 additions and 75 deletions
@ -1,53 +1,53 @@ |
||||
package org.jeecg.modules.cms.webmagic.task; |
||||
|
||||
import lombok.extern.slf4j.Slf4j; |
||||
import org.jeecg.common.util.DateUtils; |
||||
import org.jeecg.modules.cms.webmagic.service.NewsPageProcessor; |
||||
import org.jeecg.modules.cms.webmagic.service.NewsPipeline; |
||||
import org.quartz.*; |
||||
import org.springframework.beans.factory.annotation.Autowired; |
||||
import us.codecraft.webmagic.Spider; |
||||
|
||||
/** |
||||
* @Description: 同步定时任务 |
||||
* <p> |
||||
* 此处的同步是指 当定时任务的执行时间大于任务的时间间隔时 |
||||
* 会等待第一个任务执行完成才会走第二个任务 |
||||
* @author: taoyan |
||||
* @date: 2020年06月19日 |
||||
*/ |
||||
@PersistJobDataAfterExecution |
||||
@DisallowConcurrentExecution |
||||
@Slf4j |
||||
public class NewsAsyncJob implements Job { |
||||
|
||||
@Autowired |
||||
private NewsPipeline pipeline; |
||||
|
||||
@Autowired |
||||
private NewsPageProcessor pageProcessor; |
||||
|
||||
@Override |
||||
public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException { |
||||
log.info(" NewsAsyncJob Execution key:" + jobExecutionContext.getJobDetail().getKey()); |
||||
log.info(String.format("NewsAsyncJob-开始时间:" + DateUtils.getTimestamp())); |
||||
|
||||
String urlPage1 = "http://sie.huanghuai.edu.cn/index.php/item-list-category-13151.shtml"; |
||||
// String urlPageOther = "http://sie.huanghuai.edu.cn/index.php/item-list-category-13151-page-";
|
||||
// String url = "";
|
||||
try { |
||||
Spider.create(pageProcessor) |
||||
.addUrl(urlPage1) |
||||
// 抓取到的数据存数据库
|
||||
.addPipeline(pipeline) |
||||
// 开启2个线程抓取
|
||||
.thread(1) |
||||
// 异步启动爬虫
|
||||
.start(); |
||||
} catch (Exception ex) { |
||||
log.error("数据抓取异常:", ex.getMessage()); |
||||
log.error("定时抓取数据线程执行异常", ex); |
||||
} |
||||
} |
||||
|
||||
} |
||||
//package org.jeecg.modules.cms.webmagic.task;
|
||||
//
|
||||
//import lombok.extern.slf4j.Slf4j;
|
||||
//import org.jeecg.common.util.DateUtils;
|
||||
//import org.jeecg.modules.cms.webmagic.service.NewsPageProcessor;
|
||||
//import org.jeecg.modules.cms.webmagic.service.NewsPipeline;
|
||||
//import org.quartz.*;
|
||||
//import org.springframework.beans.factory.annotation.Autowired;
|
||||
//import us.codecraft.webmagic.Spider;
|
||||
//
|
||||
///**
|
||||
// * @Description: 同步定时任务
|
||||
// * <p>
|
||||
// * 此处的同步是指 当定时任务的执行时间大于任务的时间间隔时
|
||||
// * 会等待第一个任务执行完成才会走第二个任务
|
||||
// * @author: taoyan
|
||||
// * @date: 2020年06月19日
|
||||
// */
|
||||
//@PersistJobDataAfterExecution
|
||||
//@DisallowConcurrentExecution
|
||||
//@Slf4j
|
||||
//public class NewsAsyncJob implements Job {
|
||||
//
|
||||
// @Autowired
|
||||
// private NewsPipeline pipeline;
|
||||
//
|
||||
// @Autowired
|
||||
// private NewsPageProcessor pageProcessor;
|
||||
//
|
||||
// @Override
|
||||
// public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException {
|
||||
// log.info(" NewsAsyncJob Execution key:" + jobExecutionContext.getJobDetail().getKey());
|
||||
// log.info(String.format("NewsAsyncJob-开始时间:" + DateUtils.getTimestamp()));
|
||||
//
|
||||
// String urlPage1 = "http://sie.huanghuai.edu.cn/index.php/item-list-category-13151.shtml";
|
||||
//// String urlPageOther = "http://sie.huanghuai.edu.cn/index.php/item-list-category-13151-page-";
|
||||
//// String url = "";
|
||||
// try {
|
||||
// Spider.create(pageProcessor)
|
||||
// .addUrl(urlPage1)
|
||||
// // 抓取到的数据存数据库
|
||||
// .addPipeline(pipeline)
|
||||
// // 开启2个线程抓取
|
||||
// .thread(1)
|
||||
// // 异步启动爬虫
|
||||
// .start();
|
||||
// } catch (Exception ex) {
|
||||
// log.error("数据抓取异常:", ex.getMessage());
|
||||
// log.error("定时抓取数据线程执行异常", ex);
|
||||
// }
|
||||
// }
|
||||
//
|
||||
//}
|
||||
|
||||
@ -1,22 +1,22 @@ |
||||
package org.jeecg.modules.cms.webmagic.task; |
||||
|
||||
import lombok.extern.slf4j.Slf4j; |
||||
import org.jeecg.common.util.DateUtils; |
||||
import org.quartz.Job; |
||||
import org.quartz.JobExecutionContext; |
||||
import org.quartz.JobExecutionException; |
||||
|
||||
/** |
||||
* 示例不带参定时任务 |
||||
* |
||||
* @Author Scott |
||||
*/ |
||||
@Slf4j |
||||
public class SampleJobTest implements Job { |
||||
|
||||
@Override |
||||
public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException { |
||||
log.info(" >>>>>>>>SampleJobTest Execution key："+jobExecutionContext.getJobDetail().getKey()); |
||||
log.info(String.format(" 普通定时任务 SampleJobTest,时间:" + DateUtils.getTimestamp())); |
||||
} |
||||
} |
||||
//package org.jeecg.modules.cms.webmagic.task;
|
||||
//
|
||||
//import lombok.extern.slf4j.Slf4j;
|
||||
//import org.jeecg.common.util.DateUtils;
|
||||
//import org.quartz.Job;
|
||||
//import org.quartz.JobExecutionContext;
|
||||
//import org.quartz.JobExecutionException;
|
||||
//
|
||||
///**
|
||||
// * 示例不带参定时任务
|
||||
// *
|
||||
// * @Author Scott
|
||||
// */
|
||||
//@Slf4j
|
||||
//public class SampleJobTest implements Job {
|
||||
//
|
||||
// @Override
|
||||
// public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException {
|
||||
// log.info(" >>>>>>>>SampleJobTest Execution key："+jobExecutionContext.getJobDetail().getKey());
|
||||
// log.info(String.format(" 普通定时任务 SampleJobTest,时间:" + DateUtils.getTimestamp()));
|
||||
// }
|
||||
//}
|
||||
|
||||
Loading…
Reference in new issue