package org.wlld.gameRobot;

import org.wlld.MatrixTools.Matrix;

import java.util.*;

/**
 * @author lidapeng
 * @description 动态规划
 * @date 10:25 上午 2022/9/12
 */
public class DynamicProgramming {
    private List<DynamicState> dynamicStateList = new ArrayList<>();//状态集合
    private Map<Integer, Action> actionMap = new HashMap<>();//动作列表
    private List<Integer> bestStrategy = new ArrayList<>();//最佳策略
    private double gaMa = 0.9;//贴现因子
    private double valueTh = 0.0001;//价值阈值
    private int maxTimes = 400;//策略改进最大迭代次数

    public void setMaxTimes(int maxTimes) {
        this.maxTimes = maxTimes;
    }

    public void setValueTh(double valueTh) {
        this.valueTh = valueTh;
    }

    public void setGaMa(double gaMa) {
        this.gaMa = gaMa;
    }

    public List<DynamicState> getDynamicStateList() {
        return dynamicStateList;
    }

    public Map<Integer, Action> getActionMap() {
        return actionMap;
    }

    public void gameStart() {//遍历所有状态
        for (DynamicState dynamicState : dynamicStateList) {
            if (!dynamicState.isFinish()) {
                dynamicState.add();//被执行次数+1
                Map<Integer, List<DynamicState>> sonStatesMap = dynamicState.getSonStatesMap();//动作-子状态集合 被执行的时候需要修改
                int[] stateId = dynamicState.getStateId();
                for (Map.Entry<Integer, Action> actionEntry : actionMap.entrySet()) {
                    Action action = actionEntry.getValue();
                    int actionId = action.getActionId();//动作id
                    DynamicState state = getStateByStateId(action.action(stateId));//经过动作产生的新状态
                    //产生一个新的动作-子状态合集的元素
                    if (sonStatesMap.containsKey(actionId)) {//在动作集合里
                        List<DynamicState> dynamicStates = sonStatesMap.get(actionId);
                        if (!isHere(dynamicStates, state.getStateId())) {
                            dynamicStates.add(state);
                        }
                    } else {//创建动作-子状态集合
                        List<DynamicState> dynamicStateList = new ArrayList<>();
                        dynamicStateList.add(state);
                        sonStatesMap.put(actionId, dynamicStateList);
                    }
                    Map<Integer, Integer> profitMap = state.getProfitMap();//该状态的收益集合，主键是收益，值是次数 被执行的时候需要修改
                    state.add();
                    //产生一个新的收益
                    int profit = action.getProfit(stateId);
                    if (profitMap.containsKey(profit)) {
                        profitMap.put(profit, profitMap.get(profit) + 1);
                    } else {
                        profitMap.put(profit, 1);
                    }

                }
            }
        }
    }

    public Matrix getValueMatrix() throws Exception {//获取价值矩阵
        int size = dynamicStateList.size();
        int maxX = 0, maxY = 0;
        for (int i = 0; i < size; i++) {
            DynamicState dynamicState = dynamicStateList.get(i);
            int[] stateId = dynamicState.getStateId();
            int x = stateId[0];
            int y = stateId[1];
            if (x > maxX) {
                maxX = x;
            }
            if (y > maxY) {
                maxY = y;
            }
        }
        Matrix matrix = new Matrix(maxY + 1, maxX + 1);
        for (int i = 0; i < size; i++) {
            DynamicState dynamicState = dynamicStateList.get(i);
            int[] stateId = dynamicState.getStateId();
            double value = dynamicState.getValue();
            matrix.setNub(stateId[1], stateId[0], value);
        }
        return matrix;
    }

    public List<ValueFunction> getValueFunction() {//获取价值函数
        List<ValueFunction> valueFunctions = new ArrayList<>();
        int size = dynamicStateList.size();
        for (int i = 0; i < size; i++) {
            DynamicState dynamicState = dynamicStateList.get(i);
            ValueFunction valueFunction = new ValueFunction();
            valueFunction.setStateId(dynamicState.getStateId());
            valueFunction.setValue(dynamicState.getValue());
            valueFunctions.add(valueFunction);
        }
        return valueFunctions;
    }

    public List<Integer> getBestAction(int[] stateId) {//根据当前环境获取策略
        List<Integer> actions = new ArrayList<>();
        DynamicState state = getStateByStateId(stateId);//当前的环境
        if (state != null) {
            Map<Integer, List<DynamicState>> sonStatesMap = state.getSonStatesMap();
            double maxValue = 0;//最大价值
            boolean isFirstOne = true;
            for (Map.Entry<Integer, List<DynamicState>> entry : sonStatesMap.entrySet()) {
                List<DynamicState> sonStates = entry.getValue();//子状态
                double maxValue2 = 0;//actionId 的最大价值
                boolean isFirstTwo = true;
                for (DynamicState dynamicState : sonStates) {
                    double myValue = dynamicState.getValue();
                    if (myValue > maxValue2 || isFirstTwo) {
                        isFirstTwo = false;
                        maxValue2 = myValue;
                    }
                }
                if (maxValue2 > maxValue || isFirstOne) {
                    isFirstOne = false;
                    maxValue = maxValue2;
                }
            }
            //筛选等价值策略
            for (Map.Entry<Integer, List<DynamicState>> entry : sonStatesMap.entrySet()) {
                int actionId = entry.getKey();//动作id
                List<DynamicState> sonStates = entry.getValue();//子状态
                for (DynamicState dynamicState : sonStates) {
                    if (dynamicState.getValue() == maxValue) {
                        actions.add(actionId);
                        break;
                    }
                }
            }
        }
        return actions;
    }

    public void strategyStudy() {//策略学习
        //记录当前最佳策略
        int times = 0;
        boolean isDifferent = true;//策略是否不同
        do {
            int size = dynamicStateList.size();
            for (int i = 0; i < size; i++) {
                DynamicState dynamicState = dynamicStateList.get(i);
                if (!dynamicState.isFinish()) {
                    int actionId = getBestStrategyByPro(dynamicState);
                    dynamicState.setBestActionId(actionId);//通过概率获取的当前状态最佳策略
                }
            }
            if (times > 0) {
                isDifferent = compareStrategy();
            }
            if (isDifferent) {//新老策略不同，重新评估策略
                updateBestStrategy();//更新新策略
                strategyEvaluation();
            }
            times++;
        } while (isDifferent && times < maxTimes);
    }

    private boolean isHere(List<DynamicState> dynamicStates, int[] stateId) {
        boolean isHere = false;
        for (DynamicState dynamicState : dynamicStates) {
            if (Arrays.equals(dynamicState.getStateId(), stateId)) {
                isHere = true;
                break;
            }
        }
        return isHere;
    }

    private DynamicState getStateByStateId(int[] stateId) {
        DynamicState state = null;
        for (DynamicState dynamicState : dynamicStateList) {
            if (Arrays.equals(dynamicState.getStateId(), stateId)) {
                state = dynamicState;
                break;
            }
        }
        return state;
    }

    private void updateBestStrategy() {//更新最佳策略
        int size = dynamicStateList.size();
        for (int i = 0; i < size; i++) {
            DynamicState dynamicState = dynamicStateList.get(i);
            if (!dynamicState.isFinish()) {
                bestStrategy.add(dynamicState.getBestActionId());
            } else {
                bestStrategy.add(0);
            }
        }
    }

    private boolean compareStrategy() {//比较新老策略
        int size = dynamicStateList.size();
        boolean isDifferent = false;
        for (int i = 0; i < size; i++) {
            DynamicState dynamicState = dynamicStateList.get(i);
            int actionId = bestStrategy.get(i);
            if (dynamicState.getBestActionId() != actionId) {
                isDifferent = true;
                break;
            }
        }
        return isDifferent;
    }

    private int getBestStrategyByPro(DynamicState dynamicState) {//通过概率获取当前状态下的最佳策略
        Map<Integer, List<DynamicState>> sonStatesMap = dynamicState.getSonStatesMap();//动作-子状态集合
        double maxValue = 0;//最大价值
        boolean isFirst = true;
        int bestActionId = 0;//最佳动作
        for (Map.Entry<Integer, List<DynamicState>> entry : sonStatesMap.entrySet()) {
            int actionId = entry.getKey();//动作id
            double value = getValueByAction(dynamicState, actionId);//该动作的价值
            if (value > maxValue || isFirst) {
                isFirst = false;
                maxValue = value;
                bestActionId = actionId;
            }
        }
        //返回最佳动作
        return bestActionId;
    }

    private void strategyEvaluation() {//策略评估
        double maxSub;//最大差值
        do {
            maxSub = 0;
            for (DynamicState dynamicState : dynamicStateList) {//当前状态
                if (!dynamicState.isFinish()) {//非终结态
                    double sub = valueEvaluation(dynamicState);//返回一个差
                    if (sub > maxSub) {
                        maxSub = sub;
                    }
                }
            }
        } while (maxSub >= valueTh);
    }

    private double getValueByAction(DynamicState dynamicState, int actionId) {//通过动作获取价值
        Map<Integer, List<DynamicState>> sonStatesMap = dynamicState.getSonStatesMap();//动作-子状态集合
        List<DynamicState> sonStateListByAction = sonStatesMap.get(actionId);//当前状态最优策略动作下的子状态集合
        double number = dynamicState.getNumber();//当前状态被执行的次数即所有子状态被执行的总数
        double updateValue = 0;//更新价值
        for (DynamicState sonState : sonStateListByAction) {
            Map<Integer, Integer> profitMap = sonState.getProfitMap();//子状态收益集合
            double sonNumber = sonState.getNumber();//该子状态执行的次数即该子状态所有收益被执行的总次数
            double sonPro = sonNumber / number;//当前子状态在当前最优策略动作下被执行的概率
            double value = sonState.getValue() * gaMa;//该子状态的价值
            //先对r求和
            double sigmaR = 0;
            for (Map.Entry<Integer, Integer> entryProfit : profitMap.entrySet()) {
                double profit = entryProfit.getKey();//--获取profit的收益
                double profitNumber = entryProfit.getValue();//--获取profit收益的次数
                double profitPro = (profitNumber / sonNumber) * sonPro;//在当前策略动作下产生的环境，产生profit收益的概率
                double v = (value + profit) * profitPro;//价值
                sigmaR = sigmaR + v;
            }
            updateValue = updateValue + sigmaR;
        }
        return updateValue;
    }

    private double valueEvaluation(DynamicState dynamicState) {//价值评估
        double myValue = dynamicState.getValue();//当前价值
        int bestActionId = dynamicState.getBestActionId();//当前最优策略选择的动作
        double updateValue = getValueByAction(dynamicState, bestActionId);
        dynamicState.setValue(updateValue);//更新价值
        return Math.abs(myValue - updateValue);
    }
    //策略迭代
    //状态是当前环境的向量主键与价值函数
    //动作（包含动作的规则）是执行完成一个动作之后返回一个新的状态
    //策略决定执行什么动作，动作执行结束之后的收益是多少
}
