아파트 매매 거래 데이터 Bulk 수집
본문
@Override
@Transactional
public void saveAllRegionAptTradeData(String dealYmd) {
log.info("Starting to fetch and save apartment trade data for all regions, dealYmd: {}", dealYmd);
List<RegionCode> regionCodes = regionCodeRepository.findAll();
int totalRegions = regionCodes.size();
log.info("Total {} regions found", totalRegions);
// 전체 데이터를 담을 리스트
List<AptSales> totalAptSalesList = new ArrayList<>();
int totalFetched = 0;
// 1. 모든 지역의 데이터를 먼저 수집
for (RegionCode regionCode : regionCodes) {
if (regionCode != null && regionCode.getId() != null) {
String lawdCd = String.format("%05d", regionCode.getId().getGuCode());
// log.info("Fetching data for region: {} ({})", regionCode.getGu(), lawdCd);
int pageNo = 1;
int numOfRows = 1000;
while (true) {
try {
AptTradeResponse response = aptTradeFeignClient.getAptTradeDetail(
serviceKey, pageNo, numOfRows, lawdCd, dealYmd);
if (response.getBody().getItems() == null ||
response.getBody().getItems().getItemList() == null ||
response.getBody().getItems().getItemList().isEmpty()) {
break;
}
List<AptSales> currentPageItems = response.getBody().getItems().getItemList().stream()
.map(item -> AptSales.from(item, lawdCd))
.collect(Collectors.toList());
totalAptSalesList.addAll(currentPageItems);
totalFetched += currentPageItems.size();
// log.info("Fetched {} records from region {} (Total fetched: {})",
// currentPageItems.size(), regionCode.getGu(), totalFetched);
int totalCount = Integer.parseInt(response.getBody().getTotalCount());
if (pageNo * numOfRows >= totalCount) {
break;
}
pageNo++;
} catch (Exception e) {
log.error("Error fetching data for region {}: {}", lawdCd, e.getMessage());
break;
}
}
} else {
log.warn("Skipping null RegionCode or RegionCodeId");
continue;
}
}
log.info("Data fetching completed. Total records fetched: {}", totalFetched);
// 2. 수집된 전체 데이터를 2000개씩 배치 저장
try {
int batchSize = 2000;
int totalSaved = 0;
for (int i = 0; i < totalAptSalesList.size(); i += batchSize) {
int end = Math.min(i + batchSize, totalAptSalesList.size());
List<AptSales> batch = totalAptSalesList.subList(i, end);
aptSalesRepository.saveAll(batch);
totalSaved += batch.size();
log.info("Batch saved {} records. Progress: {}/{}",
batch.size(), totalSaved, totalAptSalesList.size());
}
log.info("All data saved successfully. Total records saved: {}", totalSaved);
} catch (Exception e) {
log.error("Error during batch save: {}", e.getMessage());
throw e;
}
}
@Override
@Transactional
public void saveAptTradeDataByPeriod(int startYear, int startMonth) {
LocalDate startDate = LocalDate.of(startYear, startMonth, 1);
LocalDate currentDate = LocalDate.now();
log.info("Starting to fetch and save apartment trade data from {}/{} to current date",
startYear, String.format("%02d", startMonth));
List<RegionCode> regionCodes = regionCodeRepository.findAll();
int totalRegions = regionCodes.size();
log.info("Total {} regions found", totalRegions);
// 전체 데이터를 담을 리스트
List<AptSales> totalAptSalesList = new ArrayList<>();
int totalFetched = 0;
// 1. 모든 년월에 대해 순회
LocalDate iterDate = startDate;
while (!iterDate.isAfter(currentDate)) {
String dealYmd = iterDate.format(DateTimeFormatter.ofPattern("yyyyMM"));
log.info("Processing year/month: {}", dealYmd);
// 2. 각 년월별로 모든 지역 데이터 수집
for (RegionCode regionCode : regionCodes) {
String lawdCd = String.format("%05d", regionCode.getId().getGuCode());
int pageNo = 1;
int numOfRows = 1000;
while (true) {
try {
AptTradeResponse response = aptTradeFeignClient.getAptTradeDetail(
serviceKey, pageNo, numOfRows, lawdCd, dealYmd);
if (response.getBody().getItems() == null ||
response.getBody().getItems().getItemList() == null ||
response.getBody().getItems().getItemList().isEmpty()) {
break;
}
List<AptSales> currentPageItems = response.getBody().getItems().getItemList().stream()
.map(item -> AptSales.from(item, lawdCd))
.collect(Collectors.toList());
totalAptSalesList.addAll(currentPageItems);
totalFetched += currentPageItems.size();
// 2000개가 모이면 중간 저장
if (totalAptSalesList.size() >= 2000) {
log.info("Intermediate saving of {} records (Total fetched: {})",
totalAptSalesList.size(), totalFetched);
aptSalesRepository.saveAll(totalAptSalesList);
totalAptSalesList.clear();
}
int totalCount = Integer.parseInt(response.getBody().getTotalCount());
if (pageNo * numOfRows >= totalCount) {
break;
}
pageNo++;
} catch (Exception e) {
log.error("Error fetching data for region {} in {}: {}",
lawdCd, dealYmd, e.getMessage());
break;
}
}
}
// 다음 월로 이동
iterDate = iterDate.plusMonths(1);
}
// 남은 데이터 저장
if (!totalAptSalesList.isEmpty()) {
log.info("Saving remaining {} records. Total records fetched: {}",
totalAptSalesList.size(), totalFetched);
aptSalesRepository.saveAll(totalAptSalesList);
totalAptSalesList.clear();
}
log.info("Completed saving historical apartment trade data. Total records fetched: {}",
totalFetched);
}
관련자료
-
이전
-
다음
