Skip to content

Commit

Permalink
update product service
Browse files Browse the repository at this point in the history
  • Loading branch information
lezo authored and lezo committed Feb 2, 2015
1 parent ddcaf28 commit 2f33a63
Show file tree
Hide file tree
Showing 4 changed files with 374 additions and 367 deletions.
4 changes: 2 additions & 2 deletions iscript-core/iscript-core-client/src/test/java/com/lezo/iscript/yeam/ConfigParserTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@
import com.lezo.iscript.yeam.config.ConfigClientWake;
import com.lezo.iscript.yeam.config.ConfigEtaoSimilar;
import com.lezo.iscript.yeam.config.ConfigHuihuiSigner;
import com.lezo.iscript.yeam.config.ConfigJdProduct;
import com.lezo.iscript.yeam.config.ConfigJdPromotList;
import com.lezo.iscript.yeam.config.ConfigJdPromotion;
import com.lezo.iscript.yeam.config.ConfigProxyChecker;
Expand Down Expand Up @@ -83,7 +82,7 @@ public void testConfig() throws Exception {
// parser = new ConfigBaiduDoc();
// parser = new ConfigBarCodeMatcher();
parser = new ConfigYhdProduct();
parser = new ConfigJdProduct();
// parser = new ConfigJdProduct();
// parser = new ConfigYhdPromotion();
// parser = new ConfigYhdProduct();
// parser = new ConfigYhdPromotList();
Expand Down Expand Up @@ -134,6 +133,7 @@ public void testConfig() throws Exception {
url = "http://item.yhd.com/item/9105839";
url = "http://item.jd.com/1264683.html";
url = "http://item.jd.com/1166371.html";
url = "http://item.yhd.com/item/992913";
// urlList.add(url);
TaskWritable task = new TaskWritable();
// task.put("barCode", "6900068005020");
Expand Down
4 changes: 2 additions & 2 deletions iscript-core/iscript-core-resulter/src/main/resources/spring/spring-bean-timer.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,15 +31,15 @@
<property name="limit" value="3000" />
<property name="nameList">
<list>
<value>ConfigJdProduct</value>
<!-- <value>ConfigJdProduct</value>
<value>ConfigJdPromotion</value>
<value>ConfigProxyDetector</value>
<value>ConfigYhdPromotion</value>
<value>ConfigTmallBrandShop</value>
<value>ConfigJdBrandShop</value>
<value>ConfigProxyDetector</value>
<value>ConfigProxyCollector</value>
<value>ConfigYhdProduct</value>
<value>ConfigYhdProduct</value> -->
<value>ConfigProxyChecker</value>
</list>
</property>
Expand Down
275 changes: 139 additions & 136 deletions ...awler/src/main/java/com/lezo/iscript/service/crawler/service/impl/ProductServiceImpl.java
View file
Original file line number Diff line number Diff line change
@@ -1,136 +1,139 @@
package com.lezo.iscript.service.crawler.service.impl;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;

import org.apache.commons.collections4.CollectionUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;

import com.lezo.iscript.service.crawler.dao.ProductDao;
import com.lezo.iscript.service.crawler.dto.ProductDto;
import com.lezo.iscript.service.crawler.service.ProductService;
import com.lezo.iscript.utils.BarCodeUtils;
import com.lezo.iscript.utils.BatchIterator;

@Service
public class ProductServiceImpl implements ProductService {
private static Logger logger = LoggerFactory.getLogger(ProductServiceImpl.class);
@Autowired
private ProductDao productDao;

@Override
public void batchInsertProductDtos(List<ProductDto> dtoList) {
BatchIterator<ProductDto> it = new BatchIterator<ProductDto>(dtoList);
while (it.hasNext()) {
productDao.batchInsert(it.next());
}
}

@Override
public void batchUpdateProductDtos(List<ProductDto> dtoList) {
BatchIterator<ProductDto> it = new BatchIterator<ProductDto>(dtoList);
while (it.hasNext()) {
List<ProductDto> subList = it.next();
try {
productDao.batchUpdate(subList);
} catch (Exception e) {
e.printStackTrace();
for (ProductDto dto : subList) {
logger.warn("code:{},url:{},onsail:{}", dto.getProductCode(), dto.getProductUrl(), dto.getOnsailTime());
}
}
}
}

@Override
public List<ProductDto> getProductDtos(List<String> codeList, Integer siteId) {
List<ProductDto> dtoList = new ArrayList<ProductDto>();
BatchIterator<String> it = new BatchIterator<String>(codeList);
while (it.hasNext()) {
List<ProductDto> subList = productDao.getProductDtos(it.next(), siteId);
if (CollectionUtils.isNotEmpty(subList)) {
dtoList.addAll(subList);
}
}
return dtoList;
}

public void setProductDao(ProductDao productDao) {
this.productDao = productDao;
}

@Override
public List<ProductDto> getProductDtosFromId(Long fromId, int limit, Integer siteId) {
return productDao.getProductDtosFromId(fromId, limit, siteId);
}

@Override
public void batchSaveProductDtos(List<ProductDto> dtoList) {
List<ProductDto> insertDtos = new ArrayList<ProductDto>();
List<ProductDto> updateDtos = new ArrayList<ProductDto>();
doAssort(dtoList, insertDtos, updateDtos);
batchInsertProductDtos(insertDtos);
batchUpdateProductDtos(updateDtos);
logger.info(String.format("save [%s],insert:%d,update:%d,cost:", "ProductDto", insertDtos.size(), updateDtos.size()));
}

private void doAssort(List<ProductDto> productDtos, List<ProductDto> insertDtos, List<ProductDto> updateDtos) {
Map<Integer, Set<String>> shopMap = new HashMap<Integer, Set<String>>();
Map<String, ProductDto> dtoMap = new HashMap<String, ProductDto>();
for (ProductDto dto : productDtos) {
String key = getDtoKey(dto);
ProductDto hasDto = dtoMap.get(key);
if (hasDto == null) {
dtoMap.put(key, dto);
} else if (hasDto.getUpdateTime().before(dto.getUpdateTime())) {
dtoMap.put(key, dto);
}
Set<String> codeSet = shopMap.get(dto.getSiteId());
if (codeSet == null) {
codeSet = new HashSet<String>();
shopMap.put(dto.getSiteId(), codeSet);
}
codeSet.add(dto.getProductCode());
}
for (Entry<Integer, Set<String>> entry : shopMap.entrySet()) {
List<ProductDto> hasDtos = getProductDtos(new ArrayList<String>(entry.getValue()), entry.getKey());
Set<String> hasCodeSet = new HashSet<String>();
for (ProductDto oldDto : hasDtos) {
String key = getDtoKey(oldDto);
ProductDto newDto = dtoMap.get(key);
hasCodeSet.add(oldDto.getProductCode());
convertFields(newDto, oldDto);
updateDtos.add(newDto);
}
for (String code : entry.getValue()) {
if (hasCodeSet.contains(code)) {
continue;
}
String key = entry.getKey() + "-" + code;
ProductDto newDto = dtoMap.get(key);
insertDtos.add(newDto);
}

}

}

private String getDtoKey(ProductDto dto) {
return dto.getSiteId() + "-" + dto.getProductCode();
}

private void convertFields(ProductDto newDto, ProductDto oldDto) {
newDto.setId(oldDto.getId());
if (BarCodeUtils.isBarCode(oldDto.getBarCode()) && !BarCodeUtils.isBarCode(newDto.getBarCode())) {
newDto.setBarCode(oldDto.getBarCode());
}
}
}
package com.lezo.iscript.service.crawler.service.impl;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;

import org.apache.commons.collections4.CollectionUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;

import com.lezo.iscript.service.crawler.dao.ProductDao;
import com.lezo.iscript.service.crawler.dto.ProductDto;
import com.lezo.iscript.service.crawler.service.ProductService;
import com.lezo.iscript.utils.BarCodeUtils;
import com.lezo.iscript.utils.BatchIterator;

@Service
public class ProductServiceImpl implements ProductService {
private static Logger logger = LoggerFactory.getLogger(ProductServiceImpl.class);
@Autowired
private ProductDao productDao;

@Override
public void batchInsertProductDtos(List<ProductDto> dtoList) {
BatchIterator<ProductDto> it = new BatchIterator<ProductDto>(dtoList);
while (it.hasNext()) {
productDao.batchInsert(it.next());
}
}

@Override
public void batchUpdateProductDtos(List<ProductDto> dtoList) {
BatchIterator<ProductDto> it = new BatchIterator<ProductDto>(dtoList);
while (it.hasNext()) {
List<ProductDto> subList = it.next();
try {
productDao.batchUpdate(subList);
} catch (Exception e) {
e.printStackTrace();
for (ProductDto dto : subList) {
logger.warn("code:{},url:{},onsail:{}", dto.getProductCode(), dto.getProductUrl(), dto.getOnsailTime());
}
}
}
}

@Override
public List<ProductDto> getProductDtos(List<String> codeList, Integer siteId) {
List<ProductDto> dtoList = new ArrayList<ProductDto>();
BatchIterator<String> it = new BatchIterator<String>(codeList);
while (it.hasNext()) {
List<ProductDto> subList = productDao.getProductDtos(it.next(), siteId);
if (CollectionUtils.isNotEmpty(subList)) {
dtoList.addAll(subList);
}
}
return dtoList;
}

public void setProductDao(ProductDao productDao) {
this.productDao = productDao;
}

@Override
public List<ProductDto> getProductDtosFromId(Long fromId, int limit, Integer siteId) {
return productDao.getProductDtosFromId(fromId, limit, siteId);
}

@Override
public void batchSaveProductDtos(List<ProductDto> dtoList) {
List<ProductDto> insertDtos = new ArrayList<ProductDto>();
List<ProductDto> updateDtos = new ArrayList<ProductDto>();
doAssort(dtoList, insertDtos, updateDtos);
batchInsertProductDtos(insertDtos);
batchUpdateProductDtos(updateDtos);
logger.info(String.format("save [%s],insert:%d,update:%d,cost:", "ProductDto", insertDtos.size(), updateDtos.size()));
}

private void doAssort(List<ProductDto> productDtos, List<ProductDto> insertDtos, List<ProductDto> updateDtos) {
Map<Integer, Set<String>> shopMap = new HashMap<Integer, Set<String>>();
Map<String, ProductDto> dtoMap = new HashMap<String, ProductDto>();
for (ProductDto dto : productDtos) {
String key = getDtoKey(dto);
ProductDto hasDto = dtoMap.get(key);
if (hasDto == null) {
dtoMap.put(key, dto);
} else if (hasDto.getUpdateTime().before(dto.getUpdateTime())) {
dtoMap.put(key, dto);
}
Set<String> codeSet = shopMap.get(dto.getSiteId());
if (codeSet == null) {
codeSet = new HashSet<String>();
shopMap.put(dto.getSiteId(), codeSet);
}
codeSet.add(dto.getProductCode());
}
for (Entry<Integer, Set<String>> entry : shopMap.entrySet()) {
List<ProductDto> hasDtos = getProductDtos(new ArrayList<String>(entry.getValue()), entry.getKey());
Set<String> hasCodeSet = new HashSet<String>();
for (ProductDto oldDto : hasDtos) {
String key = getDtoKey(oldDto);
ProductDto newDto = dtoMap.get(key);
hasCodeSet.add(oldDto.getProductCode());
convertFields(newDto, oldDto);
updateDtos.add(newDto);
}
for (String code : entry.getValue()) {
if (hasCodeSet.contains(code)) {
continue;
}
String key = entry.getKey() + "-" + code;
ProductDto newDto = dtoMap.get(key);
insertDtos.add(newDto);
}

}

}

private String getDtoKey(ProductDto dto) {
return dto.getSiteId() + "-" + dto.getProductCode();
}

private void convertFields(ProductDto newDto, ProductDto oldDto) {
newDto.setId(oldDto.getId());
if (BarCodeUtils.isBarCode(oldDto.getBarCode()) && !BarCodeUtils.isBarCode(newDto.getBarCode())) {
newDto.setBarCode(oldDto.getBarCode());
}
if (newDto.getShopId() == null) {
newDto.setShopId(oldDto.getShopId());
}
}
}
Loading

0 comments on commit 2f33a63

Please sign in to comment.