
package com.tmerclub.cloud.flow.service.impl;

import cn.hutool.core.collection.CollUtil;
import cn.hutool.core.date.DateUtil;
import cn.hutool.core.map.MapUtil;
import com.tmerclub.cloud.api.delivery.feign.AreaFeignClient;
import com.tmerclub.cloud.api.product.dto.ProdEffectDTO;
import com.tmerclub.cloud.api.product.feign.SpuFeignClient;
import com.tmerclub.cloud.api.user.dto.CustomerReqDTO;
import com.tmerclub.cloud.api.user.feign.UserFeignClient;
import com.tmerclub.cloud.common.cache.constant.FlowCacheNames;
import com.tmerclub.cloud.common.cache.util.RedisUtil;
import com.tmerclub.cloud.common.constant.Constant;
import com.tmerclub.cloud.common.database.dto.PageDTO;
import com.tmerclub.cloud.common.database.vo.PageVO;
import com.tmerclub.cloud.common.exception.LuckException;
import com.tmerclub.cloud.common.mongodb.util.MongoPageUtil;
import com.tmerclub.cloud.common.order.vo.AreaVO;
import com.tmerclub.cloud.common.product.constant.SpuStatus;
import com.tmerclub.cloud.common.product.vo.SpuVO;
import com.tmerclub.cloud.common.response.ServerResponseEntity;
import com.tmerclub.cloud.common.util.Arith;
import com.tmerclub.cloud.flow.bo.FlowSpuBO;
import com.tmerclub.cloud.flow.bo.FlowSpuUserBO;
import com.tmerclub.cloud.flow.constant.FlowDateTypeEnum;
import com.tmerclub.cloud.flow.constant.FlowLogPageEnum;
import com.tmerclub.cloud.flow.constant.FlowSpuSortEnum;
import com.tmerclub.cloud.flow.constant.FlowVisitEnum;
import com.tmerclub.cloud.flow.service.SpuAnalyseMongoService;
import com.tmerclub.cloud.flow.util.FlowUtil;
import com.tmerclub.cloud.flow.util.IpSearchUtil;
import com.tmerclub.cloud.flow.vo.ProductFlowInfoVO;
import com.tmerclub.cloud.flow.vo.ShopFlowInfoVO;
import org.apache.dubbo.config.annotation.DubboReference;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.Sort;
import org.springframework.data.mongodb.core.BulkOperations;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.data.mongodb.core.aggregation.*;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Service;

import java.util.*;
import java.util.function.Function;
import java.util.stream.Collectors;


/**
 * 流量分析—用户访问数据
 *
 * @author YXF
 * @date 2021-05-21 15:25:19
 */
@Service
public class SpuAnalyseMongoServiceImpl implements SpuAnalyseMongoService {

    private static final Logger LOG = LoggerFactory.getLogger(SpuAnalyseMongoServiceImpl.class);

    @Autowired
    private MongoTemplate mongoTemplate;
    @DubboReference
    private AreaFeignClient areaFeignClient;
    @Autowired
    private MongoPageUtil mongoPageUtil;
    @DubboReference
    private SpuFeignClient spuFeignClient;
    @DubboReference
    private UserFeignClient userFeignClient;

    private static final int SPU_STATISTICS_TIME = 2 * 60 * 1000;

    @Override
    public void statisticsAndSaveSpuUserOfDay(Date beginDate, Date endDate) {
        // 用户id、省id、商品id、店铺id，商品浏览量
        List<FlowSpuUserBO> spuAnalyseLogList = this.statisticsSpuUserOfDay(beginDate, endDate);

        // 根据oid来分组. 获取用户id, 新用户，用户/访客
        List<Long> newUserIds = userFeignClient.listNewUserIdByTime(new CustomerReqDTO(beginDate, endDate));

        Map<Long, List<FlowSpuUserBO>> map = spuAnalyseLogList.stream().collect(Collectors.groupingBy(FlowSpuUserBO::getOid));
        map.forEach((oid, list) -> {
            List<Long> userIds = list.stream().map(FlowSpuUserBO::getUserId).filter(Objects::nonNull).collect(Collectors.toList());
            Long userId = oid;
            // 新用户也算是新访客
            int newVisit = 1;
            if (CollUtil.isNotEmpty(userIds) && Objects.nonNull(userIds.get(0))) {
                userId = userIds.get(0);
                if (!newUserIds.contains(userId)) {
                    newVisit = 0;
                }
            }
            for (FlowSpuUserBO flowSpuUserBO : list) {
                flowSpuUserBO.setUserId(userId);
                flowSpuUserBO.setNewVisit(newVisit);
                flowSpuUserBO.setOid(null);
            }
        });

        if (CollUtil.isEmpty(spuAnalyseLogList)) {
            return;
        }

        // 获取省名称列表
        List<AreaVO> areaList = areaFeignClient.listProvinceArea();
        Map<String, Long> areaMap = areaList.stream().collect(Collectors.toMap(AreaVO::getAreaName, AreaVO::getAreaId));

        int index = 1;
        Date currentTime = new Date();
        String date = DateUtil.format(beginDate, "yyMMdd");

        List<Long> spuIds = spuAnalyseLogList.stream().map(FlowSpuUserBO::getSpuId).collect(Collectors.toList());

        ServerResponseEntity<List<SpuVO>> spuResponseEntity = spuFeignClient.listSpuBySpuIds(spuIds);

        if (!spuResponseEntity.isSuccess()) {
            throw new LuckException(spuResponseEntity.getMsg());
        }
        Map<Long, SpuVO> spuMap;
        if (CollUtil.isEmpty(spuResponseEntity.getData())) {
            spuMap = Collections.emptyMap();
        } else {
            spuMap = spuResponseEntity.getData().stream().collect(Collectors.toMap(SpuVO::getSpuId, s -> s));
        }

        for (FlowSpuUserBO flowSpuUserBO : spuAnalyseLogList) {
            flowSpuUserBO.setId(FlowUtil.getId(date, index++));
            flowSpuUserBO.setCreateTime(beginDate);
            flowSpuUserBO.setUpdateTime(currentTime);
            if (Objects.nonNull(flowSpuUserBO.getPlusShopCartNum()) && flowSpuUserBO.getPlusShopCartNum() > 0) {
                flowSpuUserBO.setPlusShopCart(1);
            }
            if (Objects.nonNull(flowSpuUserBO.getVisitNum()) && flowSpuUserBO.getVisitNum() > 0) {
                flowSpuUserBO.setVisit(1);
            }
            // 设置省id
            flowSpuUserBO.setProvinceId(IpSearchUtil.getProvinceIdByUserIp(flowSpuUserBO.getProvinceId(), areaMap));

            if (spuMap.containsKey(flowSpuUserBO.getSpuId())) {
                SpuVO spuVO = spuMap.get(flowSpuUserBO.getSpuId());
                if (Objects.nonNull(spuVO)) {
                    flowSpuUserBO.setShopId(spuVO.getShopId());
                    flowSpuUserBO.setSupplierSpuId(spuVO.getSupplierSpuId());
                    flowSpuUserBO.setSupplierId(spuVO.getSupplierId());
                }
            }
        }

        // 批量插入
        this.batchInsertFlowSpuUserBO(beginDate, endDate, spuAnalyseLogList);
    }

    @Override
    public void statisticsAndSaveSpuOfDay(Integer dateType, Date beginDate, Date endDate) {
        // 获取商品id列表
        // 获取商品总数量
        List<FlowSpuBO> flowSpuList = this.listSpuStatisticsOfDay(beginDate, endDate, null);
        LOG.info("statisticsAndSaveSpuOfDay-统计商品总数量:{}", flowSpuList.size());
        if (CollUtil.isEmpty(flowSpuList)) {
            return;
        }
        // 批量插入
        this.batchInsertFlowSpuBO(dateType, beginDate, endDate, flowSpuList);
    }

    @Override
    public List<FlowSpuBO> listSpuStatisticsOfDay(Date beginDate, Date endDate, List<Long> spuIds) {
        // 插入商品数据
        Map<Long, FlowSpuBO> spuStatisticsMap = new HashMap<>(15);

        // 曝光次数、曝光人数、加购人数、加购件数
        LOG.info("listSpuStatisticsOfDay-统计曝光次数、曝光人数、加购人数、加购件数");
        this.statisticsSpuFlowOfDay(beginDate, endDate, spuStatisticsMap, spuIds);

        // 统计订单和退款信息
        LOG.info("listSpuStatisticsOfDay-统计订单和退款信息");
        this.statisticsSpuOrderAndRefundInfo(beginDate, endDate, spuStatisticsMap, spuIds);

        //
        if (MapUtil.isEmpty(spuStatisticsMap)) {
            LOG.info("listSpuStatisticsOfDay spuStatisticsMap is empty");
            return new ArrayList<>(0);
        }

        if (CollUtil.isNotEmpty(spuIds)) {
            LOG.info("listSpuStatisticsOfDay-删除指定范围外的商品");
            spuStatisticsMap.keySet().removeIf(spuId -> !spuIds.contains(spuId));
        }

        // 插入商品的店铺id、供应商商品id、供应商id
        LOG.info("listSpuStatisticsOfDay-插入商品的店铺id、供应商商品id、供应商id");
        this.setSpuInfo(spuStatisticsMap);
        return new ArrayList<>(spuStatisticsMap.values());
    }

    @Override
    public void statisticsAndSaveSpuOfTime(int dateType, Date beginDate, Date endDate) {
        // 统计商品的曝光次数、曝光人数、加购人数、加购件数
        Map<Long, FlowSpuBO> spuStatisticsMap = this.statisticsSpuFlowOfTime(beginDate, endDate);
        // 统计商品的订单和退款信息
        this.statisticsSpuOrderAndRefundInfo(beginDate, endDate, spuStatisticsMap, null);
        // 插入商品的店铺id、供应商商品id、供应商id
        this.setSpuInfo(spuStatisticsMap);
        List<FlowSpuBO> flowSpuBOList = new ArrayList<>(spuStatisticsMap.values());
        if (CollUtil.isEmpty(flowSpuBOList)) {
            return;
        }

        // 批量保存
        this.batchInsertFlowSpuBO(dateType, beginDate, endDate, flowSpuBOList);
    }

    @Override
    public PageVO<FlowSpuBO> pageSpuEffect(PageDTO pageDTO, ProdEffectDTO prodEffectDTO) {
        // 类型为今日实时，要替换为当天，保存时使用的类型为日，表中没有今日实时的类型
        if (Objects.equals(prodEffectDTO.getDateType(), FlowDateTypeEnum.REAL_TIME.value())) {
            prodEffectDTO.setDateType(FlowDateTypeEnum.DAY.value());
        }
        Long beginId = FlowUtil.stringToId(FlowUtil.dateToString(prodEffectDTO.getStartTime()), FlowSpuBO.ID_LENGTH);
        Long endId = FlowUtil.stringToId(FlowUtil.dateToString(prodEffectDTO.getEndTime()), FlowSpuBO.ID_LENGTH);

        Criteria criteria = Criteria.where("_id").gte(beginId).lt(endId).and("dateType").is(prodEffectDTO.getDateType());

        if (Objects.nonNull(prodEffectDTO.getShopId())) {
            criteria.and("shopId").is(prodEffectDTO.getShopId());
        }


        Query query = new Query(criteria);
        if (Objects.nonNull(prodEffectDTO.getSortParam())) {
            Sort.Direction direction;
            if (Objects.equals(prodEffectDTO.getSortType(), 1)) {
                direction = Sort.Direction.ASC;
            } else {
                direction = Sort.Direction.DESC;
            }

            query.with(Sort.by(direction, FlowSpuSortEnum.getFieldName(prodEffectDTO.getSortParam())));
        }
        LOG.info("pageSpuEffect-query:{}", query);
        return mongoPageUtil.doPage(FlowSpuBO.class, pageDTO, query);
    }

    @Override
    public void statisticsRealSpuData() {
        Date endTime = new Date();
        Date beginTime = DateUtil.beginOfDay(endTime);

        // 获取上次统计的时间
        String timeValue = RedisUtil.get(FlowCacheNames.SPU_STATISTICS_TIME);
        Date cacheTime = null;
        if (Objects.nonNull(timeValue)) {
            long timeMillis = Long.parseLong(timeValue);
            if (System.currentTimeMillis() - timeMillis < SPU_STATISTICS_TIME) {
                LOG.info("流量分析-商品洞察的今日实时统计时间小于两分钟，不进行统计");
                return;
            }
            // 缓存时间为当天时间时，初始化cacheTime
            if (timeMillis >= beginTime.getTime()) {
                LOG.info("初始化cacheTime：赋值为上一次统计缓存的时间");
                cacheTime = new Date(timeMillis);
            }
        }

        // ******************** 时间不存在或者不是当天，统计当天全部的数据 ********************
        if (Objects.isNull(cacheTime)) {
            LOG.info("商品洞察实时统计当天全部的数据");
            endTime = DateUtil.offsetDay(beginTime, 1);
            this.statisticsAndSaveSpuOfDay(FlowDateTypeEnum.DAY.value(), beginTime, endTime);
        }
        //  ******************** 时间大于当天的开始时间，统计缓存时间到当前的数据 ********************
        else {
            LOG.info("商品洞察实时统计刚操作的数据");
            // 统计结束时间为第二天开始时间
            Date statisticsEndDate= DateUtil.beginOfDay(DateUtil.offsetDay(beginTime, 1));
            this.statisticsLatestData(statisticsEndDate, beginTime, cacheTime);
        }
        // 添加统计时间的缓存数据，缓存时间为一天
        RedisUtil.set(FlowCacheNames.SPU_STATISTICS_TIME, String.valueOf(System.currentTimeMillis()), Constant.DAY_SECOND);
    }

    @Override
    public ShopFlowInfoVO countVisitUserInPastTenDays(Date startTime, Date endTime, Long supplierId) {
        Criteria criteria = Criteria.where("_id")
                    .gte(FlowUtil.timeToId(startTime))
                    .lt(FlowUtil.timeToId(endTime))
                .and("supplierId").is(supplierId);

        GroupOperation userGroupOperation = Aggregation.group("userId")
                .max("newVisit").as("newVisit");

        GroupOperation groupOperation = Aggregation.group("newVisit")
                .last("newVisit").as("newVisit")
                .count().as("visitNum");

        TypedAggregation<FlowSpuUserBO> typedAggregation = Aggregation.newAggregation(
                FlowSpuUserBO.class,
                Aggregation.match(criteria),
                userGroupOperation,
                groupOperation
        );
        List<FlowSpuUserBO> flowAnalysisList = mongoTemplate
                .aggregate(typedAggregation, FlowSpuUserBO.class)
                .getMappedResults();

        Map<Integer, Integer> map;
        if (CollUtil.isNotEmpty(flowAnalysisList)) {
            flowAnalysisList = flowAnalysisList.stream().filter(flowSpuUserBO -> Objects.nonNull(flowSpuUserBO.getNewVisit())).collect(Collectors.toList());
            map = flowAnalysisList.stream().collect(Collectors.toMap(FlowSpuUserBO::getNewVisit, FlowSpuUserBO::getVisitNum));
        } else {
            map = Collections.emptyMap();
        }

        ShopFlowInfoVO shopFlowInfoVO = new ShopFlowInfoVO();
        shopFlowInfoVO.setOldVisitUserCount(map.getOrDefault(0, 0));
        shopFlowInfoVO.setNewVisitUserCount(map.getOrDefault(1, 0));
        shopFlowInfoVO.setVisitUserCount(shopFlowInfoVO.getOldVisitUserCount() + shopFlowInfoVO.getNewVisitUserCount());
        return shopFlowInfoVO;
    }

    @Override
    public Long countProductVisitCount(Date startTime, Date endTime, Long supplierId) {
        Criteria criteria = Criteria.where("_id")
                .gte(FlowUtil.timeToId(startTime, FlowSpuBO.ID_LENGTH))
                .lt(FlowUtil.timeToId(endTime, FlowSpuBO.ID_LENGTH))
                .and("supplierId").is(supplierId)
                .and("dateType").is(FlowDateTypeEnum.DAY.value());
        TypedAggregation<FlowSpuBO> typedAggregation = Aggregation.newAggregation(
                FlowSpuBO.class,
                Aggregation.match(criteria),
                Aggregation.group().sum("visit").as("visit")

        );
        List<FlowSpuBO> flowAnalysisList = mongoTemplate
                .aggregate(typedAggregation, "flow_spu", FlowSpuBO.class)
                .getMappedResults();
        if (CollUtil.isEmpty(flowAnalysisList)) {
            return 0L;
        }
        return flowAnalysisList.get(0).getVisit();
    }

    @Override
    public List<ShopFlowInfoVO> listShopRankIngByFlow(Date startTime, Date endTime, Integer limit) {
        Criteria criteria = Criteria.where("_id")
                    .gte(FlowUtil.timeToId(startTime, FlowSpuBO.ID_LENGTH))
                    .lt(FlowUtil.timeToId(endTime, FlowSpuBO.ID_LENGTH))
                .and("dateType").is(FlowDateTypeEnum.DAY.value());

        GroupOperation userGroupOperation = Aggregation.group("shopId", "userId")
                .first("shopId").as("shopId")
                .sum("visit").as("visit");

        GroupOperation groupOperation = Aggregation.group("shopId")
                .first("shopId").as("shopId")
                .sum("visit").as("visitCount")
                .count().as("visitUserCount");

        TypedAggregation<FlowSpuBO> typedAggregation = Aggregation.newAggregation(
                FlowSpuBO.class,
                Aggregation.match(criteria),
                userGroupOperation,
                groupOperation,
                Aggregation.sort(Sort.Direction.DESC, "visitCount"),
                Aggregation.limit(limit)
        );
        List<ShopFlowInfoVO> flowAnalysisList = mongoTemplate
                .aggregate(typedAggregation, "flow_spu", ShopFlowInfoVO.class)
                .getMappedResults();
        if (CollUtil.isEmpty(flowAnalysisList)) {
            return new ArrayList<>(0);
        }

        return new ArrayList<>(flowAnalysisList);
    }

    @Override
    public List<ProductFlowInfoVO> listSupplierProdRankIngByFlow(Date startTime, Date endTime, Long supplierId) {
        Criteria criteria = Criteria.where("_id")
                    .gte(FlowUtil.timeToId(startTime, FlowSpuBO.ID_LENGTH))
                    .lt(FlowUtil.timeToId(endTime, FlowSpuBO.ID_LENGTH))
                .and("dateType").is(FlowDateTypeEnum.DAY.value())
                .and("supplierId").is(supplierId);

        // 商品和访问用户的去重
        GroupOperation userGroupOperation = Aggregation.group("spuId", "userId")
                .first("spuId").as("spuId");

        // 商品的去重
        GroupOperation groupOperation = Aggregation.group("spuId")
                .first("spuId").as("spuId")
                .count().as("visitUserCount");

        TypedAggregation<ProductFlowInfoVO> typedAggregation = Aggregation.newAggregation(
                ProductFlowInfoVO.class,
                Aggregation.match(criteria),
                userGroupOperation,
                groupOperation,
                Aggregation.sort(Sort.Direction.DESC, "visitUserCount"),
                Aggregation.limit(10));

        List<ProductFlowInfoVO> flowAnalysisList = mongoTemplate
                .aggregate(typedAggregation, "flow_spu", ProductFlowInfoVO.class)
                .getMappedResults();

        List<Long> spuIds = flowAnalysisList.stream().map(ProductFlowInfoVO::getSpuId).filter(Objects::nonNull).collect(Collectors.toList());
        if (CollUtil.isEmpty(flowAnalysisList) || CollUtil.isEmpty(spuIds)) {
            return new ArrayList<>(0);
        }

        List<ProductFlowInfoVO> orderPayList = this.listProductFlowInfoList(startTime, endTime, spuIds);

        Map<Long, Long> payMap = orderPayList.stream().collect(Collectors.toMap(ProductFlowInfoVO::getSpuId, ProductFlowInfoVO::getPayUserCount));

        for (ProductFlowInfoVO productFlowInfoVO : flowAnalysisList) {
            productFlowInfoVO.setPayUserCount(payMap.getOrDefault(productFlowInfoVO.getSpuId(), Constant.ZERO_LONG));
        }

        return flowAnalysisList;
    }

    private List<ProductFlowInfoVO> listProductFlowInfoList(Date startTime, Date endTime, List<Long> spuIds) {


        Criteria criteria = Criteria.where("createTime")
                .gte(startTime)
                .lt(endTime)
                .and("isPayed").is(1)
                .and("orderItems.spuId").in(spuIds);

        ProjectionOperation projectionOperation = Aggregation.project("userId").and("orderItems.spuId").as("spuId");

        // 商品和访问用户的去重
        GroupOperation userGroupOperation = Aggregation.group("spuId", "userId")
                .first("spuId").as("spuId");

        // 商品的去重
        GroupOperation groupOperation = Aggregation.group("spuId")
                .first("spuId").as("spuId")
                .count().as("payUserCount");

        TypedAggregation<ProductFlowInfoVO> typedAggregation = Aggregation.newAggregation(
                ProductFlowInfoVO.class,
                Aggregation.match(criteria),
                Aggregation.unwind("orderItems"),
                projectionOperation,
                userGroupOperation,
                groupOperation);

        return mongoTemplate
                .aggregate(typedAggregation, "order", ProductFlowInfoVO.class)
                .getMappedResults();
    }

    private void statisticsLatestData(Date endTime, Date beginTime, Date cacheTime) {
        // 1.统计时间段内数据
        // 获取需要统计的商品id列表
        List<Long> spuIds = this.listSpuId(cacheTime, endTime);
        if (CollUtil.isEmpty(spuIds)) {
            return;
        }
        // 统计商品数据
        List<FlowSpuBO> flowSpuList = this.listSpuStatisticsOfDay(beginTime, endTime, spuIds);
        if (CollUtil.isEmpty(flowSpuList)) {
            return;
        }

        String date = DateUtil.format(beginTime, "yyMMdd");
        Long beginId = FlowUtil.timeToId(beginTime, FlowSpuBO.ID_LENGTH);
        Long endId = FlowUtil.timeToId(endTime, FlowSpuBO.ID_LENGTH);
        Query idQuery = new Query(Criteria.where("dateType").is(FlowDateTypeEnum.DAY.value()).and("_id").gte(beginId).lt(endId));
        idQuery.with(Sort.by(Sort.Order.desc("_id"))).limit(1);
        // 获取最大的id，然后递增
        List<FlowSpuBO> flowSpus = mongoTemplate.find(idQuery, FlowSpuBO.class);
        long index;
        if (CollUtil.isNotEmpty(flowSpus) && Objects.nonNull(flowSpus.get(0).getId())) {
            index = flowSpus.get(0).getId();
        } else {
            index = FlowUtil.getId(date, 0, FlowSpuBO.ID_LENGTH);
        }

        // 完善统计的数据
        for (FlowSpuBO flowSpuBO : flowSpuList) {
            flowSpuBO.setCreateTime(beginTime);
            flowSpuBO.setUpdateTime(endTime);
            flowSpuBO.setDateType(FlowDateTypeEnum.DAY.value());
        }


        // 2.查询已存在的商品数据id
        Criteria criteria = Criteria.where("dateType").is(FlowDateTypeEnum.DAY.value())
                .and("_id").gte(FlowUtil.timeToId(beginTime, FlowDateTypeEnum.DAY.value(), FlowSpuBO.ID_LENGTH))
                .and("spuId").in(spuIds);
        Query query = new Query(criteria);
        query.fields().include("_id", "spuId");
        List<FlowSpuBO> spuList = mongoTemplate.find(query, FlowSpuBO.class);


        // 3.插入新统计数据
        // 没有已保存的数据，就全部插入
        if (CollUtil.isEmpty(spuList)) {
            for (FlowSpuBO spus : flowSpuList) {
                spus.setId(++index);
            }
            mongoTemplate.insertAll(flowSpuList);
            return;
        }
        // 有部分数据已保存到表中，不能直接批量插入，只能通过BulkOperations批量更新或者插入数据
        Map<Long, FlowSpuBO> spuMap = spuList.stream().collect(Collectors.toMap(FlowSpuBO::getSpuId, Function.identity()));
        Set<Long> ids = spuMap.keySet();
        BulkOperations bulkOperations = mongoTemplate.bulkOps(BulkOperations.BulkMode.UNORDERED, FlowSpuBO.class);

        Iterator<FlowSpuBO> iterator = flowSpuList.iterator();
        while (iterator.hasNext()) {
            FlowSpuBO flowSpuBO = iterator.next();
            // 表中不存在的商品数据->插入
            if (!ids.contains(flowSpuBO.getSpuId())) {
                flowSpuBO.setId(++index);
                continue;
            }
            // 表中存在的商品数据->根据id来替换
            FlowSpuBO flowSpuDb = spuMap.get(flowSpuBO.getSpuId());
            flowSpuBO.setId(flowSpuDb.getId());
            bulkOperations.replaceOne(new Query(Criteria.where("_id").is(flowSpuBO.getId())), flowSpuBO);
            // 移除替换的数据
            iterator.remove();
        }
        // 如果有剩余的数据，就添加批量插入操作
        if (CollUtil.isNotEmpty(flowSpuList)) {
            bulkOperations.insert(flowSpuList);
        }
        // 执行批量操作
        bulkOperations.execute();
    }

    private List<Long> listSpuId(Date beginDate, Date endDate) {

        // 订单关联
        Criteria orderCriteria = Criteria.where(null).orOperator(
                Criteria.where("createTime").gte(beginDate).lt(endDate),
                Criteria.where("payTime").gte(beginDate).lt(endDate)
        );

        ProjectionOperation orderProjectionOperation = Aggregation.project("_id")
                .and("orderItems.spuId").as("spuId");

        UnionWithOperation orderUnionWithOperation = UnionWithOperation
                .unionWith("order")
                .pipeline(
                        Aggregation.match(orderCriteria),
                        Aggregation.unwind("orderItems"),
                        orderProjectionOperation,
                        Aggregation.group("spuId").first("spuId").as("spuId")
                );

        // 退款订单关联
        Criteria refundCriteria = Criteria.where("_id").gt(0).orOperator(Criteria.where("createTime").gte(beginDate).lt(endDate), Criteria.where("refundTime").gte(beginDate).lt(endDate));

        ProjectionOperation refundProjectionOperation = Aggregation.project("_id")
                .and("orderItems.spuId").as("spuId");
        UnionWithOperation refundUnionWithOperation = UnionWithOperation
                .unionWith("order_refund")
                .pipeline(
                        Aggregation.match(refundCriteria),
                        Aggregation.unwind("orderItems"),
                        refundProjectionOperation,
                        Aggregation.group("spuId").first("spuId").as("spuId")
                );
        TypedAggregation<FlowSpuBO> typedAggregation = Aggregation.newAggregation(
                FlowSpuBO.class,
                Aggregation.match(Criteria.where("dt").gte(beginDate).lt(endDate).and("pid").is(FlowLogPageEnum.PROD_INFO.value())),
                Aggregation.group("bid").first("bid").as("spuId"),
//                Aggregation.project("_id").and("bid").as("spuId"),
                Aggregation.group("spuId").first("spuId").as("spuId"),
                orderUnionWithOperation,
                refundUnionWithOperation,
                Aggregation.group("spuId").first("spuId").as("spuId")

        );
        LOG.info("listSpuId.typedAggregation:{}", typedAggregation);
        List<FlowSpuBO> flowAnalysisList = mongoTemplate
                .aggregate(typedAggregation, "flow_user_log", FlowSpuBO.class)
                .getMappedResults();
        if (CollUtil.isEmpty(flowAnalysisList)) {
            return new ArrayList<>();
        }

        return flowAnalysisList.stream().map(FlowSpuBO::getSpuId).collect(Collectors.toList());
    }

    private Map<Long, FlowSpuBO> statisticsSpuFlowOfTime(Date beginDate, Date endDate) {
        // 曝光次数、曝光人数、加购人数、加购件数
        Long beginId = FlowUtil.timeToId(beginDate);
        Long endId = FlowUtil.timeToId(endDate);

        // 分组，统计商品的用户浏览量
        GroupOperation userGroupOperation = Aggregation.group("spuId", "userId")
                .first("spuId").as("spuId")
                .max("visit").as("visit")
                .sum("visitNum").as("visitNum")
                .max("plusShopCart").as("plusShopCart")
                .sum("plusShopCartNum").as("plusShopCartNum");


        GroupOperation groupOperation = Aggregation.group("spuId")
                .first("spuId").as("spuId")
                .sum("visit").as("visitUser")
                .sum("visitNum").as("visit")
                .sum("plusShopCart").as("plusShopCartUser")
                .sum("plusShopCartNum").as("plusShopCart");

        TypedAggregation<FlowSpuBO> typedAggregation = Aggregation.newAggregation(
                FlowSpuBO.class,
                Aggregation.match(Criteria.where("_id").gte(beginId).lt(endId)),
                userGroupOperation,
                groupOperation
        );
        LOG.info("statisticsSpuFlowOfTime:typedAggregation：{}", typedAggregation);

        List<FlowSpuBO> spuStatisticsList = mongoTemplate
                .aggregate(typedAggregation, "flow_spu_user", FlowSpuBO.class)
                .getMappedResults();
        Map<Long, FlowSpuBO> spuStatisticsMap = new HashMap<>();
        for (FlowSpuBO flowSpuBO : spuStatisticsList) {
            spuStatisticsMap.put(flowSpuBO.getSpuId(), flowSpuBO);
        }
        return spuStatisticsMap;
    }

    private void statisticsSpuOrderAndRefundInfo(Date beginDate, Date endDate, Map<Long, FlowSpuBO> spuStatisticsMap, List<Long> spuIds) {
        // 下单人数、下单件数、下单金额、支付人数、支付件数、支付金额
        this.statisticsSpuOrderInfo(beginDate, endDate, spuStatisticsMap, spuIds);

        // 申请退款订单数、申请退款人数、申请退款金额
        this.statisticsSpuRefundInfo(beginDate, endDate, spuStatisticsMap, spuIds, true);

        // 成功退款订单数、成功退款人数、退款金额
        this.statisticsSpuRefundInfo(beginDate, endDate, spuStatisticsMap, spuIds, false);

        // 计算退款率
        for (FlowSpuBO flowSpuBO : spuStatisticsMap.values()) {
            if (Objects.isNull(flowSpuBO.getSuccessRefundUser())) {
                flowSpuBO.setRefundRate(0D);
                continue;
            }
            if (Objects.isNull(flowSpuBO.getPayUser())) {
                flowSpuBO.setRefundRate(100D);
                continue;
            }
            double refundRate = Arith.mul(Arith.div(flowSpuBO.getSuccessRefundUser(), flowSpuBO.getPayUser(), 4), 100);
            flowSpuBO.setRefundRate(refundRate);
        }
    }

    private void statisticsSpuFlowOfDay(Date beginDate, Date endDate, Map<Long, FlowSpuBO> spuStatisticsMap, List<Long> spuIds) {
        // 曝光次数、曝光人数
        List<AggregationOperation> aggregationList = new ArrayList<>();
        Criteria visitCriteria = Criteria.where("dt").gte(beginDate).lt(endDate)
                .and("pid").is(FlowLogPageEnum.PROD_INFO.value());
        if (CollUtil.isNotEmpty(spuIds)) {
            visitCriteria.and("bid").in(spuIds);
        }
        // 过滤
        aggregationList.add(Aggregation.match(visitCriteria));

        // 分组，统计商品的用户浏览量
        aggregationList.add(Aggregation.group("bid", "oid")
                .first("bid").as("spuId")
                .count().as("visit"));

        // 分组，统计商品的浏览量和浏览用户数
        aggregationList.add(Aggregation.group("bid")
                .first("spuId").as("spuId")
                .sum("visit").as("visit")
                .count().as("visitUser"));

        List<FlowSpuBO> visitSpuStatisticsList = loadSpuStatisticsByFlowUserLog(aggregationList);

        for (FlowSpuBO flowSpuBO : visitSpuStatisticsList) {
            spuStatisticsMap.put(flowSpuBO.getSpuId(), flowSpuBO);
        }

        aggregationList.clear();


        Criteria criteria = Criteria.where("dt").gte(beginDate).lt(endDate)
                .and("ve").is(FlowVisitEnum.SHOP_CAT.value());
        if (CollUtil.isNotEmpty(spuIds)) {
            criteria.and("bid").in(spuIds);
        }
        // 加购人数、加购件数
        aggregationList.add(Aggregation.match(criteria));

        // 分组，统计商品的用户加购件数
        aggregationList.add(Aggregation.group("bid", "uid")
                .first("bid").as("spuId")
                .count().as("plusShopCart"));

        // 分组，统计商品的加购人数、加购件数
        aggregationList.add(Aggregation.group("bid")
                .first("spuId").as("spuId")
                .sum("plusShopCart").as("plusShopCart")
                .count().as("plusShopCartUser"));

        List<FlowSpuBO> plusShopCartSpuStatisticsList = this.loadSpuStatisticsByFlowUserLog(aggregationList);

        for (FlowSpuBO flowSpuBO : plusShopCartSpuStatisticsList) {
            if (spuStatisticsMap.containsKey(flowSpuBO.getSpuId())) {
                FlowSpuBO visitFlowSpuBO = spuStatisticsMap.get(flowSpuBO.getSpuId());
                visitFlowSpuBO.setPlusShopCart(flowSpuBO.getPlusShopCart());
                visitFlowSpuBO.setPlusShopCartUser(flowSpuBO.getPlusShopCartUser());
            } else {
                spuStatisticsMap.put(flowSpuBO.getSpuId(), flowSpuBO);
            }
        }
    }

    private void statisticsSpuRefundInfo(Date beginDate, Date endDate, Map<Long, FlowSpuBO> spuStatisticsMap, List<Long> spuIds, boolean applyRefund) {
        // 申请退款订单数、申请退款人数、申请退款金额、成功退款订单数、成功退款人数、退款金额
//        List<AggregationOperation> aggregationList = new ArrayList<>();
        String timeStr;
        if (applyRefund) {
            timeStr = "createTime";
        } else {
            timeStr = "refundTime";
        }

        Criteria criteria = Criteria.where(timeStr).gte(beginDate).lt(endDate);

        if (CollUtil.isNotEmpty(spuIds)) {
            criteria.and("orderItems.spuId").in(spuIds);
        }

        ProjectionOperation projectionOperation = Aggregation.project("userId")
                .and("orderItems.spuId").as("spuId")
                .and("orderItems.count").as("applyRefundOrder")
                .and("orderItems.refundAmount").as("applyRefundAmount");

        GroupOperation userGroupOperation = Aggregation.group("spuId", "userId")
                .first("spuId").as("spuId")
                .sum("applyRefundOrder").as("applyRefundOrder")
                .sum("applyRefundAmount").as("applyRefundAmount");

        GroupOperation groupOperation = Aggregation.group("spuId")
                .first("spuId").as("spuId")
                .sum("applyRefundOrder").as("applyRefundOrder")
                .sum("applyRefundAmount").as("applyRefundAmount")
                .count().as("applyRefundUser");

        TypedAggregation<FlowSpuBO> typedAggregation = Aggregation.newAggregation(
                FlowSpuBO.class,
                Aggregation.match(criteria),
                Aggregation.unwind("orderItems"),
                projectionOperation,
                userGroupOperation,
                groupOperation
        );

        List<FlowSpuBO> spuStatisticsList = mongoTemplate
                .aggregate(typedAggregation, "order_refund", FlowSpuBO.class)
                .getMappedResults();

        // 插入订单数据
        for (FlowSpuBO flowSpuBO : spuStatisticsList) {
            FlowSpuBO spuStatistics = spuStatisticsMap.get(flowSpuBO.getSpuId());
            if (Objects.isNull(spuStatistics)) {
                spuStatisticsMap.put(flowSpuBO.getSpuId(), flowSpuBO);
                spuStatistics = flowSpuBO;
            }
            if (applyRefund) {
                spuStatistics.setApplyRefundUser(flowSpuBO.getApplyRefundUser());
                spuStatistics.setApplyRefundOrder(flowSpuBO.getApplyRefundOrder());
                spuStatistics.setApplyRefundAmount(flowSpuBO.getApplyRefundAmount());
            } else {
                spuStatistics.setSuccessRefundUser(flowSpuBO.getApplyRefundUser());
                spuStatistics.setSuccessRefundOrder(flowSpuBO.getApplyRefundOrder());
                spuStatistics.setSuccessRefundAmount(flowSpuBO.getApplyRefundAmount());
            }
            spuStatistics.setPlaceOrderNum(flowSpuBO.getPlaceOrderNum());
            spuStatistics.setPlaceOrderAmount(flowSpuBO.getPlaceOrderAmount());
            spuStatistics.setPayNum(flowSpuBO.getPayNum());
            spuStatistics.setPayAmount(flowSpuBO.getPayAmount());
            spuStatistics.setPayUser(flowSpuBO.getPayUser());
        }
    }

    private List<FlowSpuBO> loadSpuStatisticsByFlowUserLog(List<AggregationOperation> aggregationList) {
        TypedAggregation<FlowSpuBO> typedAggregation = Aggregation.newAggregation(FlowSpuBO.class, aggregationList);
        LOG.info("loadSpuStatisticsByFlowUserLog aggregationList:{}", typedAggregation);
        return mongoTemplate
                .aggregate(typedAggregation, "flow_user_log", FlowSpuBO.class)
                .getMappedResults();
    }

    private void statisticsSpuOrderInfo(Date beginDate, Date endDate, Map<Long, FlowSpuBO> spuStatisticsMap, List<Long> spuIds) {
        // 下单人数、下单件数、下单金额、支付人数、支付件数、支付金额
        List<AggregationOperation> aggregationList = new ArrayList<>();
        Criteria criteria = Criteria.where("createTime").gte(beginDate).lt(endDate);
        if (CollUtil.isNotEmpty(spuIds)) {
            criteria.and("orderItems.spuId").in(spuIds);
        }
        // 过滤
        aggregationList.add(Aggregation.match(criteria));
        // 展开
        aggregationList.add(Aggregation.unwind("orderItems"));

        aggregationList.add(Aggregation.project("userId", "isPayed")
                .and("orderItems.spuId").as("spuId")
                .and("orderItems.actualTotal").as("placeOrderAmount")
                .and("orderItems.count").as("placeOrderNum")
                .and(ConditionalOperators.when(Criteria.where("isPayed").is(1)).then("$orderItems.actualTotal").otherwise(0)).as("payAmount")
                .and(ConditionalOperators.when(Criteria.where("isPayed").is(1)).then("$orderItems.count").otherwise(0)).as("payNum")
                .and(ConditionalOperators.when(Criteria.where("isPayed").is(1)).then(1).otherwise(0)).as("payUser")
        );

        // 根据用户和商品id分组，统计商品的用户数据
        aggregationList.add(Aggregation.group("spuId", "userId")
                .sum("placeOrderAmount").as("placeOrderAmount")
                .sum("placeOrderNum").as("placeOrderNum")
                .sum("payAmount").as("payAmount")
                .sum("payNum").as("payNum")
                .sum("payUser").as("payUser")
                .count().as("placeOrderUser"));


        // 分组，统计商品的订单数据
        aggregationList.add(Aggregation.group("_id.spuId")
                .first("_id.spuId").as("spuId")
                .sum("placeOrderAmount").as("placeOrderAmount")
                .sum("placeOrderNum").as("placeOrderNum")
                .sum("payAmount").as("payAmount")
                .sum("payNum").as("payNum")
                .sum("payUser").as("payUser")
                .sum("placeOrderUser").as("placeOrderUser"));

        TypedAggregation<FlowSpuBO> typedAggregation = Aggregation.newAggregation(
                FlowSpuBO.class,
                aggregationList);
        LOG.info("statisticsSpuOrderInfo.typedAggregation:{}", typedAggregation);

        List<FlowSpuBO> spuStatisticsList = mongoTemplate
                .aggregate(typedAggregation, "order", FlowSpuBO.class)
                .getMappedResults();

        // 插入订单数据
        for (FlowSpuBO flowSpuBO : spuStatisticsList) {
            if (spuStatisticsMap.containsKey(flowSpuBO.getSpuId())) {
                FlowSpuBO spuStatistics = spuStatisticsMap.get(flowSpuBO.getSpuId());
                spuStatistics.setPlaceOrderUser(flowSpuBO.getPlaceOrderUser());
                spuStatistics.setPlaceOrderNum(flowSpuBO.getPlaceOrderNum());
                spuStatistics.setPlaceOrderAmount(flowSpuBO.getPlaceOrderAmount());
                spuStatistics.setPayNum(flowSpuBO.getPayNum());
                spuStatistics.setPayAmount(flowSpuBO.getPayAmount());
                spuStatistics.setPayUser(flowSpuBO.getPayUser());
            } else {
                spuStatisticsMap.put(flowSpuBO.getSpuId(), flowSpuBO);
            }
        }
    }

    private void batchInsertFlowSpuBO(Integer dateType, Date beginDate, Date endDate, List<FlowSpuBO> flowSpuUserList) {
        String date = DateUtil.format(beginDate, "yyMMdd");
        Date currentTime = new Date();

        Long beginId = FlowUtil.timeToId(beginDate, FlowSpuBO.ID_LENGTH);
        Long endId = FlowUtil.timeToId(endDate, FlowSpuBO.ID_LENGTH);
        Query idQuery = new Query(Criteria.where("_id").gte(beginId).lt(endId));
        idQuery.with(Sort.by(Sort.Order.desc("_id"))).limit(1);
        // 获取最大的id，然后递增
        List<FlowSpuBO> flowSpus = mongoTemplate.find(idQuery, FlowSpuBO.class);
        long index = 0;
        if (CollUtil.isNotEmpty(flowSpus) && Objects.nonNull(flowSpus.get(0).getId())) {
            index = flowSpus.get(0).getId();
        } else {
            index = FlowUtil.getId(date, 0, FlowSpuBO.ID_LENGTH);
        }

        for (FlowSpuBO flowSpuBO : flowSpuUserList) {
            index ++;
            flowSpuBO.setCreateTime(beginDate);
            flowSpuBO.setUpdateTime(currentTime);
            flowSpuBO.setDateType(dateType);
            flowSpuBO.setId(index);
        }
        Query query = FlowUtil.getRemoveQuery(beginDate, endDate, FlowSpuBO.ID_LENGTH, dateType);
        LOG.info("batchInsertFlowSpuBO query:{}", query);
        // 先删除再更新
        mongoTemplate.remove(query, FlowSpuBO.class);

        // 统计的数据批量保存到flow_spu
        mongoTemplate.insertAll(flowSpuUserList);
    }

    private void setSpuInfo(Map<Long, FlowSpuBO> spuStatisticsMap) {
        ServerResponseEntity<List<SpuVO>> spuResponseEntity = spuFeignClient.listSpuBySpuIds(new ArrayList<>(spuStatisticsMap.keySet()));
        if (!spuResponseEntity.isSuccess()) {
            throw new LuckException(spuResponseEntity.getMsg());
        }
        if (CollUtil.isEmpty(spuResponseEntity.getData())) {
            return;
        }

        Map<Long, SpuVO> spuMap = spuResponseEntity.getData().stream().collect(Collectors.toMap(SpuVO::getSpuId, s -> s));

        Iterator<Long> iterator = spuStatisticsMap.keySet().iterator();

        while (iterator.hasNext()) {
            Long spuId = iterator.next();
            SpuVO spuVO = spuMap.get(spuId);
            // 删除的商品不统计
            if (!spuMap.containsKey(spuId) || Objects.equals(spuVO.getStatus(), SpuStatus.DELETE.value())) {
                iterator.remove();
                continue;
            }
            FlowSpuBO flowSpuBO = spuStatisticsMap.get(spuId);
            flowSpuBO.setShopId(spuVO.getShopId());
            flowSpuBO.setSupplierSpuId(spuVO.getSupplierSpuId());
            flowSpuBO.setSupplierId(spuVO.getSupplierId());
        }
    }


    private void batchInsertFlowSpuUserBO(Date beginDate, Date endDate, List<FlowSpuUserBO> flowSpuUserList) {
        Long beginId = FlowUtil.timeToId(beginDate);
        Long endId = FlowUtil.timeToId(endDate);
        // 先删除再更新-mongo没有事务，如果之前保存后程序报错了，数据还是保存在数据库中
        mongoTemplate.remove(new Query(Criteria.where("_id").gte(beginId).lt(endId)), FlowSpuUserBO.class);

        // 统计的数据批量保存到flow_system
        mongoTemplate.insertAll(flowSpuUserList);
    }

    private List<FlowSpuUserBO> statisticsSpuUserOfDay(Date beginDate, Date endDate) {
        MatchOperation matchOperation = Aggregation.match(Criteria.where("dt").gte(beginDate).lt(endDate)
                .orOperator(Criteria.where("pid").is(FlowLogPageEnum.PROD_INFO.value()),
                        Criteria.where("ve").is(FlowVisitEnum.SHOP_CAT.value()))
        );

        // 根据oid和bid分组，初步去重
        GroupOperation initGroupOperation = Aggregation.group("oid", "bid")
                .first("ip").as("ip")
                .last("uid").as("uid")
                .last("bid").as("bid")
                .sum("n").as("plusShopCartNum")
                .count().as("visitNum");

        GroupOperation groupOperation = Aggregation.group("uid", "bid")
                .first("ip").as("provinceId")
                .last("uid").as("userId")
                .last("oid").as("oid")
                .last("bid").as("spuId")
                .sum("plusShopCartNum").as("plusShopCartNum")
                .sum("visitNum").as("visitNum");

        TypedAggregation<FlowSpuUserBO> typedAggregation = Aggregation.newAggregation (
                FlowSpuUserBO.class,
                matchOperation,
                initGroupOperation,
                groupOperation,
                Aggregation.project("provinceId", "userId", "spuId", "plusShopCartNum", "visitNum", "oid").andExclude("_id")
        );

        LOG.info("statisticsSpuUserOfDay.typedAggregation:{}", typedAggregation);

        return mongoTemplate
                .aggregate(typedAggregation, "flow_user_log", FlowSpuUserBO.class)
                .getMappedResults();
    }
}
