项目中使用了mongodb来存储热点文章的url和一些信息
1.maven坐标
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.3.2</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-collections4</artifactId>
<version>4.0</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-collections4</artifactId>
<version>4.0</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.3</version>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.4</version>
</dependency>
<dependency>
<groupId>commons-fileupload</groupId>
<artifactId>commons-fileupload</artifactId>
<version>1.3.1</version>
</dependency>
<dependency>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
<version>1.10</version>
</dependency>
<dependency>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
<version>1.1.3</version>
</dependency>
<dependency>
<groupId>commons-beanutils</groupId>
<artifactId>commons-beanutils</artifactId>
<version>1.9.2</version>
</dependency>
<!-- mongodb -->
<dependency>
<groupId>org.springframework.data</groupId>
<artifactId>spring-data-mongodb</artifactId>
<version>1.7.2.RELEASE</version>
<dependency>
Spring配置
<!-- mongoDB -->
<bean id="mongoClientURI" class="com.mongodb.MongoClientURI">
<constructor-arg value="${mongodb.uri}" />
</bean>
<bean id="mongoDbFactory" class="org.springframework.data.mongodb.core.SimpleMongoDbFactory">
<constructor-arg ref="mongoClientURI" />
</bean>
<mongo:mapping-converter id="converter" />
<bean id="mongo" class="org.springframework.data.mongodb.core.MongoTemplate">
<constructor-arg ref="mongoDbFactory" />
<constructor-arg ref="converter" />
</bean>
mongodb操作字符
/**
* MongoDB keywords for various query operations
*
*/
public class QueryOperators {
public static final String OR = "$or";
public static final String AND = "$and";
public static final String GT = "$gt";
public static final String GTE = "$gte";
public static final String LT = "$lt";
public static final String LTE = "$lte";
public static final String NE = "$ne";
public static final String IN = "$in";
public static final String NIN = "$nin";
public static final String MOD = "$mod";
public static final String ALL = "$all";
public static final String SIZE = "$size";
public static final String EXISTS = "$exists";
public static final String ELEM_MATCH = "$elemMatch";
// (to be implemented in QueryBuilder)
public static final String WHERE = "$where";
public static final String NOR = "$nor";
public static final String TYPE = "$type";
public static final String NOT = "$not";
// geo operators
public static final String WITHIN = "$within";
public static final String NEAR = "$near";
public static final String NEAR_SPHERE = "$nearSphere";
public static final String BOX = "$box";
public static final String CENTER = "$center";
public static final String POLYGON = "$polygon";
public static final String CENTER_SPHERE = "$centerSphere";
// (to be implemented in QueryBuilder)
public static final String MAX_DISTANCE = "$maxDistance";
public static final String UNIQUE_DOCS = "$uniqueDocs";
// meta query operators (to be implemented in QueryBuilder)
public static final String RETURN_KEY = "$returnKey";
public static final String MAX_SCAN = "$maxScan";
public static final String ORDER_BY = "$orderby";
public static final String EXPLAIN = "$explain";
public static final String SNAPSHOT = "$snapshot";
public static final String MIN = "$min";
public static final String MAX = "$max";
public static final String SHOW_DISK_LOC = "$showDiskLoc";
public static final String HINT = "$hint";
public static final String COMMENT = "$comment";
}
2. MongoDao.java 公用mongodb类可继承
import java.lang.reflect.Field;
import javax.annotation.Resource;
import org.apache.commons.beanutils.BeanUtils;
import org.springframework.data.mongodb.core.MongoOperations;
import com.mongodb.BasicDBObject;
import com.mongodb.DBObject;
public abstract class MongoDao {
private static final String ID_COLLECTION = "id_generator";
@Resource
protected MongoOperations mongo;
public Long nextId(String collectionName) {
DBObject obj =
mongo.getCollection(ID_COLLECTION).findAndModify(new BasicDBObject("coll", collectionName), null, null,
false, new BasicDBObject("$inc", new BasicDBObject("id", 1L)), true, true);
return (Long) obj.get("id");
}
public static <T> T toEntity(DBObject dbo, Class<T> entityClazz) {
if (dbo == null) {
return null;
}
T entity = null;
try {
entity = entityClazz.newInstance();
Field[] fields = entityClazz.getDeclaredFields();
for (Field field : fields) {
String fn = field.getName();
Object fv = dbo.get(fn);
if (fv != null) {
BeanUtils.setProperty(entity, fn, fv);
}
}
} catch (Exception e) {
e.printStackTrace();
}
return entity;
}
public static DBObject buildFields(String... fieldNames) {
DBObject fields = new BasicDBObject();
for (String fn : fieldNames) {
fields.put(fn, 1);
}
return fields;
}
}
3.HotArticleDao.java
public interface HotArticleDao {
void insert(HotArticle hotArticle);
void update(HotArticle hotArticle);
public void updateByUrlMd5(HotArticle hotArticle);
List<HotArticle> findListByParam(Date beginTime, Date endTime,Integer siteId,Integer sourceId,String url,Pagination page);
Long findCount(Date beginTime, Date endTime,Integer siteId,Integer sourceId,String url);
void deleteAllBefore(Date time);
}
4.HotArticleDaoImpl mongdb具体操作
@Repository
public class HotArticleDaoImpl extends MongoDao implements HotArticleDao {
protected static final String COLLECTION = "sp_hotarticle";
//保存
@Override
public void insert(HotArticle hotArticle) {
hotArticle.setArtId(nextId(COLLECTION));
mongo.insert(hotArticle,COLLECTION);
}
//根据id更新
@Override
public void update(HotArticle hotArticle) {
DBObject query = new BasicDBObject("artId", hotArticle.getArtId());
DBObject o = new BasicDBObject();
mongo.getConverter().write(hotArticle, o);
mongo.getCollection(COLLECTION).update(query, new BasicDBObject("$set", o));
//有则更新,没有则插入
//mongo.getCollection(COLLECTION).update(query, new BasicDBObject("$set", o),true,true);
}
//更具url md5更新
@Override
public void updateByUrlMd5(HotArticle hotArticle) {
DBObject query = new BasicDBObject("urlMd5", hotArticle.getUrlMd5());
DBObject o = new BasicDBObject();
mongo.getConverter().write(hotArticle, o);
mongo.getCollection(COLLECTION).update(query, new BasicDBObject("$set", o));
}
//分页查询,带条件
@Override
public List<HotArticle> findListByParam(Date beginTime, Date endTime,Integer siteId,Integer sourceId,String url,Pagination page){
endTime=DateUtils.addDays(endTime, 1);
DBObject query =
new BasicDBObject("createdTime",
new BasicDBObject(QueryOperators.GTE, beginTime).append(QueryOperators.LTE,endTime));
if (sourceId != null && sourceId !=0) {
query.put("sourceId", sourceId);
}
if (siteId != null && siteId !=0) {
query.put("siteId", siteId);
}
if (StringUtils.isNotBlank(url)) {
query.put("url", Pattern.compile("^.*" + url + ".*$", Pattern.CASE_INSENSITIVE));
}
DBObject fields =
buildFields("artId","siteId","hotSourceId","sourceUrl","newId", "url", "urlMd5","createdTime","status");
DBCursor cursor =
mongo.getCollection(COLLECTION).find(query, fields).skip(page.getStartItem())
.limit(page.getPageSize()).sort(new BasicDBObject("createdTime",-1));
List<HotArticle> result = new LinkedList<HotArticle>();
while (cursor.hasNext()) {
DBObject o = cursor.next();
result.add(toEntity(o, HotArticle.class));
}
return result;
}
//删除一个时间点以前的数据
@Override
public void deleteAllBefore(Date time) {
DBObject query =
new BasicDBObject("createdTime",
new BasicDBObject(QueryOperators.LT, time));
mongo.getCollection(COLLECTION).remove(query);
}
//统计
@Override
public Long findCount(Date beginTime, Date endTime, Integer siteId, Integer sourceId, String url) {
endTime=DateUtils.addDays(endTime, 1);
DBObject query =
new BasicDBObject("createdTime",
new BasicDBObject(QueryOperators.GTE, beginTime).append(QueryOperators.LTE,
endTime));
if (sourceId != null) {
query.put("sourceId", sourceId);
}
if (siteId != null && siteId !=0) {
query.put("siteId", siteId);
}
if (StringUtils.isNotBlank(url)) {
query.put("url", Pattern.compile("^.*" + url + ".*$", Pattern.CASE_INSENSITIVE));
}
Long count = mongo.getCollection(COLLECTION).count(query);
return count;
}
}
5.HotArticle.java 实体类
public class HotArticle implements Serializable {
private static final long serialVersionUID = -312854320019484371L;
private Long artId; // mongdb自增id
private Integer siteId; // 网站id
private Integer hotSourceId;// 热点二级内容源id
private String sourceUrl;
private Long newId; // 在sp_content表中id
private String url; // 热点文章url
private String urlMd5; // url md5
private Date createdTime; // 创建时间
private Integer status; //0,未抓取,1,已抓取
public String getUrl() {
return url;
}
public void setUrl(String url) {
this.url = url;
}
public Date getCreatedTime() {
return createdTime;
}
public void setCreatedTime(Date createdTime) {
this.createdTime = createdTime;
}
public String getUrlMd5() {
return urlMd5;
}
public void setUrlMd5(String urlMd5) {
this.urlMd5 = urlMd5;
}
public Long getNewId() {
return newId;
}
public void setNewId(Long newId) {
this.newId = newId;
}
public Long getArtId() {
return artId;
}
public void setArtId(Long artId) {
this.artId = artId;
}
public Integer getSiteId() {
return siteId;
}
public void setSiteId(Integer siteId) {
this.siteId = siteId;
}
public Integer getStatus() {
return status;
}
public void setStatus(Integer status) {
this.status = status;
}
public String getSourceUrl() {
return sourceUrl;
}
public void setSourceUrl(String sourceUrl) {
this.sourceUrl = sourceUrl;
}
public Integer getHotSourceId() {
return hotSourceId;
}
public void setHotSourceId(Integer hotSourceId) {
this.hotSourceId = hotSourceId;
}
}