zhc077
2 years ago
parent
e3f6118c44
commit
2dbc5500d3
2 changed files with 75 additions and 75 deletions
@ -1,53 +1,53 @@ |
|||||||
package org.jeecg.modules.cms.webmagic.task; |
//package org.jeecg.modules.cms.webmagic.task;
|
||||||
|
//
|
||||||
import lombok.extern.slf4j.Slf4j; |
//import lombok.extern.slf4j.Slf4j;
|
||||||
import org.jeecg.common.util.DateUtils; |
//import org.jeecg.common.util.DateUtils;
|
||||||
import org.jeecg.modules.cms.webmagic.service.NewsPageProcessor; |
//import org.jeecg.modules.cms.webmagic.service.NewsPageProcessor;
|
||||||
import org.jeecg.modules.cms.webmagic.service.NewsPipeline; |
//import org.jeecg.modules.cms.webmagic.service.NewsPipeline;
|
||||||
import org.quartz.*; |
//import org.quartz.*;
|
||||||
import org.springframework.beans.factory.annotation.Autowired; |
//import org.springframework.beans.factory.annotation.Autowired;
|
||||||
import us.codecraft.webmagic.Spider; |
//import us.codecraft.webmagic.Spider;
|
||||||
|
//
|
||||||
/** |
///**
|
||||||
* @Description: 同步定时任务 |
// * @Description: 同步定时任务
|
||||||
* <p> |
// * <p>
|
||||||
* 此处的同步是指 当定时任务的执行时间大于任务的时间间隔时 |
// * 此处的同步是指 当定时任务的执行时间大于任务的时间间隔时
|
||||||
* 会等待第一个任务执行完成才会走第二个任务 |
// * 会等待第一个任务执行完成才会走第二个任务
|
||||||
* @author: taoyan |
// * @author: taoyan
|
||||||
* @date: 2020年06月19日 |
// * @date: 2020年06月19日
|
||||||
*/ |
// */
|
||||||
@PersistJobDataAfterExecution |
//@PersistJobDataAfterExecution
|
||||||
@DisallowConcurrentExecution |
//@DisallowConcurrentExecution
|
||||||
@Slf4j |
//@Slf4j
|
||||||
public class NewsAsyncJob implements Job { |
//public class NewsAsyncJob implements Job {
|
||||||
|
//
|
||||||
@Autowired |
// @Autowired
|
||||||
private NewsPipeline pipeline; |
// private NewsPipeline pipeline;
|
||||||
|
//
|
||||||
@Autowired |
// @Autowired
|
||||||
private NewsPageProcessor pageProcessor; |
// private NewsPageProcessor pageProcessor;
|
||||||
|
//
|
||||||
@Override |
// @Override
|
||||||
public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException { |
// public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException {
|
||||||
log.info(" NewsAsyncJob Execution key:" + jobExecutionContext.getJobDetail().getKey()); |
// log.info(" NewsAsyncJob Execution key:" + jobExecutionContext.getJobDetail().getKey());
|
||||||
log.info(String.format("NewsAsyncJob-开始时间:" + DateUtils.getTimestamp())); |
// log.info(String.format("NewsAsyncJob-开始时间:" + DateUtils.getTimestamp()));
|
||||||
|
//
|
||||||
String urlPage1 = "http://sie.huanghuai.edu.cn/index.php/item-list-category-13151.shtml"; |
// String urlPage1 = "http://sie.huanghuai.edu.cn/index.php/item-list-category-13151.shtml";
|
||||||
// String urlPageOther = "http://sie.huanghuai.edu.cn/index.php/item-list-category-13151-page-";
|
//// String urlPageOther = "http://sie.huanghuai.edu.cn/index.php/item-list-category-13151-page-";
|
||||||
// String url = "";
|
//// String url = "";
|
||||||
try { |
// try {
|
||||||
Spider.create(pageProcessor) |
// Spider.create(pageProcessor)
|
||||||
.addUrl(urlPage1) |
// .addUrl(urlPage1)
|
||||||
// 抓取到的数据存数据库
|
// // 抓取到的数据存数据库
|
||||||
.addPipeline(pipeline) |
// .addPipeline(pipeline)
|
||||||
// 开启2个线程抓取
|
// // 开启2个线程抓取
|
||||||
.thread(1) |
// .thread(1)
|
||||||
// 异步启动爬虫
|
// // 异步启动爬虫
|
||||||
.start(); |
// .start();
|
||||||
} catch (Exception ex) { |
// } catch (Exception ex) {
|
||||||
log.error("数据抓取异常:", ex.getMessage()); |
// log.error("数据抓取异常:", ex.getMessage());
|
||||||
log.error("定时抓取数据线程执行异常", ex); |
// log.error("定时抓取数据线程执行异常", ex);
|
||||||
} |
// }
|
||||||
} |
// }
|
||||||
|
//
|
||||||
} |
//}
|
||||||
|
|||||||
@ -1,22 +1,22 @@ |
|||||||
package org.jeecg.modules.cms.webmagic.task; |
//package org.jeecg.modules.cms.webmagic.task;
|
||||||
|
//
|
||||||
import lombok.extern.slf4j.Slf4j; |
//import lombok.extern.slf4j.Slf4j;
|
||||||
import org.jeecg.common.util.DateUtils; |
//import org.jeecg.common.util.DateUtils;
|
||||||
import org.quartz.Job; |
//import org.quartz.Job;
|
||||||
import org.quartz.JobExecutionContext; |
//import org.quartz.JobExecutionContext;
|
||||||
import org.quartz.JobExecutionException; |
//import org.quartz.JobExecutionException;
|
||||||
|
//
|
||||||
/** |
///**
|
||||||
* 示例不带参定时任务 |
// * 示例不带参定时任务
|
||||||
* |
// *
|
||||||
* @Author Scott |
// * @Author Scott
|
||||||
*/ |
// */
|
||||||
@Slf4j |
//@Slf4j
|
||||||
public class SampleJobTest implements Job { |
//public class SampleJobTest implements Job {
|
||||||
|
//
|
||||||
@Override |
// @Override
|
||||||
public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException { |
// public void execute(JobExecutionContext jobExecutionContext) throws JobExecutionException {
|
||||||
log.info(" >>>>>>>>SampleJobTest Execution key："+jobExecutionContext.getJobDetail().getKey()); |
// log.info(" >>>>>>>>SampleJobTest Execution key："+jobExecutionContext.getJobDetail().getKey());
|
||||||
log.info(String.format(" 普通定时任务 SampleJobTest,时间:" + DateUtils.getTimestamp())); |
// log.info(String.format(" 普通定时任务 SampleJobTest,时间:" + DateUtils.getTimestamp()));
|
||||||
} |
// }
|
||||||
} |
//}
|
||||||
|
|||||||
Loading…
Reference in new issue