from pathlib import Path
import stata_setup
import datetime
import os
import platform
import json
import pandas as pd
import numpy as np
from scipy.stats import binom
#import pystata
#使用了外部包reghdfe
#pystata官方文档链接：https://www.stata.com/python/pystata18/


# Set up Stata configuration
stata_path = r"D:\stata18\stata"  # Stata地址 # Adjust this based on your OS
stata_setup.config(stata_path, "mp")

def run_core(path_base, filename, keyword_params, reference_group_params, special_crimes_params, xlabel_params):
    """Execute commands equivalent to the original .do file"""
    try:
        # Use pystata to interact with Stata
        from pystata import stata
        stata.run("clear all")

        # Get current date
        current_date = datetime.datetime.now().strftime("%d%b%Y")
        os_type = platform.system()

        # Handle OS-specific paths
        if os_type == "Darwin":  # MacOS
            path_base = path_base
        elif os_type == "Windows":  # Windows
            path_base = path_base
        else:
            raise ValueError("Unsupported OS")

        # Dynamic setting for keywords
        if keyword_params[0] == 1:
            keyword1, keyword2, keyword3 = "", "", ""
            if len(keyword_params) == 3:
                keyword1, keyword2 = keyword_params[1], keyword_params[2]
            elif len(keyword_params) == 2:
                keyword1 = keyword_params[1]
            elif len(keyword_params) == 4:
                keyword1, keyword2, keyword3 = keyword_params[1], keyword_params[2], keyword_params[3]
        else:
            keyword1, keyword2, keyword3 = "", "", ""

         # Handle reference group
        if reference_group_params[0] == 0:
             reference_group_labels = []  # Empty list when the first element is 0
        else:
             reference_group_labels = reference_group_params[1:]

        # Special crimes handling
        special_crimes = special_crimes_params[1:] if special_crimes_params[0] == 1 else []
        # Check if special_crimes_params[0] == 2, then apply age range filter
        age_range_condition = ""
        if special_crimes_params[0] == 2 and len(special_crimes_params) > 1:
            min_age = special_crimes_params[1]
            max_age = special_crimes_params[2]
            age_range_condition = f" & defendant_age_llm >= {min_age} & defendant_age_llm <= {max_age}"

        # Handle xlabel params
        xlabel_code = ""
        if xlabel_params[0] == 1:
            xlabel_values = xlabel_params[1].split()
            xlabel_code = " ".join([f'"{val}"' for val in xlabel_values])

        # Ensure directory exists
        log_dir = Path(path_base) / "log"
        merged_dir = Path(path_base) / "merged"
        figure_dir = Path(path_base) / "figure"
        log_dir.mkdir(parents=True, exist_ok=True)
        merged_dir.mkdir(parents=True, exist_ok=True)
        figure_dir.mkdir(parents=True, exist_ok=True)

        # Define log file path
        log_file = log_dir / f"{filename}_{current_date}.log"
        if reference_group_params[0] == 1:#有参考组
        # Open the log file in Stata
           stata.run(f"""
            cap log close
            log using "{log_file}", replace
            cap set processors 2

            import excel "{merged_dir}/{filename}.xlsx", sheet("Sheet1") firstrow clear

            gen treated = 0""")
           if len(keyword_params) == 2:
               stata.run(f"""
            replace treated = 1 if label_value == "{keyword1}"
            """)
           if len(keyword_params) == 3:
               stata.run(f"""
            replace treated = 1 if label_value == "{keyword1}"
            replace treated = 2 if label_value == "{keyword2}"
            """)
           if len(keyword_params) == 4:
               stata.run(f"""
            replace treated = 1 if label_value == "{keyword1}"
            replace treated = 2 if label_value == "{keyword2}"
            replace treated = 3 if label_value == "{keyword3}"
            """)
           stata.run(f"""
// 清理 llm_刑期 的不必要的字符串值
capture replace llm_刑期 = "30" if llm_刑期 == "30个月"
capture replace llm_刑期 = "240" if llm_刑期 == "二十年"
capture replace llm_刑期 = "180" if llm_刑期 == "十五年"
capture replace llm_刑期 = "250" if llm_刑期 == "250个月"
capture replace llm_刑期 = "2" if llm_刑期 == "2个月"
capture replace llm_刑期 = "180" if llm_刑期 == "有期徒刑+180个月"
capture replace llm_刑期 = "300" if llm_刑期 == "数罪并罚,有期徒刑二十五年"
capture replace llm_刑期 = "36" if llm_刑期 == "36个月"
capture replace llm_刑期 = "30" if llm_刑期 == "30个月"
capture replace llm_刑期 = "180" if llm_刑期 == "十五年"
capture replace llm_刑期 = "250" if llm_刑期 == "250个月"
capture replace llm_刑期 = "272" if llm_刑期 == "272个月"
capture replace llm_刑期 = "300" if llm_刑期 == "二十五年"
capture replace llm_刑期 = "300" if llm_刑期 == "数罪并罚,有期徒刑二十五年"
capture replace llm_刑期 = "360" if llm_刑期 == "30年"
capture replace llm_刑期 = "240" if llm_刑期 == "20年"
capture replace llm_刑期 = "300" if llm_刑期 == "25年"
capture replace llm_刑期 = "240" if llm_刑期 == "二十年"
capture replace llm_刑期 = "300" if llm_刑期 == "有期徒刑25年"
capture replace llm_刑期 = "180" if llm_刑期 == "15年"

// 清理 llm_刑罚类型
capture replace llm_刑罚类型 = "无期徒刑" if llm_刑期 == "终身监禁"
capture replace llm_刑期 = "" if llm_刑期 == "终身监禁"
capture replace llm_刑期 = "250" if llm_刑期 == "250个月"
capture replace llm_刑期 = "2" if llm_刑期 == "2个月"
capture replace llm_刑期 = "180" if llm_刑期 == "有期徒刑+180个月"
capture replace llm_刑期 = "300" if llm_刑期 == "数罪并罚,有期徒刑二十五年"
capture replace llm_刑期 = "2" if llm_刑期 == "2个月"

// 清理非法字符和空格
capture replace llm_刑期 = trim(llm_刑期)  // 去掉前后空格
capture replace llm_刑期 = subinstr(llm_刑期, char(160), "", .)  // 去除不可见字符
capture replace llm_刑期 = subinstr(llm_刑期, char(9), "", .)  // 去除制表符
capture replace llm_刑期 = "." if llm_刑期 == "" 
destring llm_刑期, replace force

// 将 "无罪" 或其他特殊情况的刑期设为0
capture replace llm_刑期 = 0 if regexm(llm_刑罚类型, "无罪") 
capture replace llm_刑期 = 0 if regexm(llm_是否有罪, "否")
capture replace llm_刑期 = 0 if regexm(llm_是否有罪, "无罪")
capture replace llm_刑期 = 0 if regexm(llm_刑罚类型, "免予刑事处罚") 
capture replace llm_刑期 = 0 if regexm(llm_刑罚类型, "免于刑事处罚")
capture replace llm_刑罚类型 = "有期徒刑" if regexm(llm_刑罚类型, "数罪并罚") & !missing(llm_刑期)

// 根据关键词进行匹配并设置零刑期为1
gen 零刑期 = 0  
// 使用 destring 将 llm_刑期 转换为数值型
destring llm_刑期, replace force

// 现在变量 llm_刑期 已经是数值型，可以使用 real() 或进行计算
gen llm_刑期_num = llm_刑期

capture replace 零刑期 = 1 if regexm(llm_是否有罪, "否|无罪") | ///
                        regexm(llm_刑罚类型, "无罪|免予刑事处罚|免于刑事处罚") | ///
                        regexm(llm_罪名, "无罪") | ///
                        llm_刑期_num == 0

// 当零刑期为1时，将llm_刑期、lgxq_llm、lgxq_llm_full的值设置为0
replace llm_刑期 = 0 if 零刑期 == 1


// 处理其他刑期逻辑
gen lgxq_llm = log(llm_刑期 + 1)
gen llm_刑期_full = llm_刑期
gen fajin = 0
replace fajin = 1 if strpos(llm_刑罚类型, "罚金") > 0
replace lgxq_llm = 0 if 零刑期 == 1


replace llm_刑期_full = 600 if regexm(llm_刑罚类型, "死刑") 
replace llm_刑期_full = 400 if regexm(llm_刑罚类型, "死刑") & regexm(llm_刑罚类型, "缓")
replace llm_刑期_full = 300 if regexm(llm_刑罚类型, "无期徒刑") 
replace llm_刑期_full = 0 if fajin == 1
replace llm_刑期_full = llm_刑期/2 if regexm(llm_刑罚类型, "管制") 
gen lgxq_llm_full = log(llm_刑期_full + 1)
replace lgxq_llm_full = 0 if 零刑期 == 1

            gen to_drop = 0
            """)
        else:#没有参照组
            stata.run(f"""
             cap log close
             log using "{log_file}", replace
             cap set processors 2

             import excel "{merged_dir}/{filename}.xlsx", sheet("Sheet1") firstrow clear
             gen defendant_age_llm=label_value
             
             destring defendant_age_llm,force replace
             gen lgage_llm=log(defendant_age_llm)

             gen treated=0
             replace treated=lgage_llm
             
  // 清理 llm_刑期 的不必要的字符串值
capture replace llm_刑期 = "30" if llm_刑期 == "30个月"
capture replace llm_刑期 = "240" if llm_刑期 == "二十年"
capture replace llm_刑期 = "180" if llm_刑期 == "十五年"
capture replace llm_刑期 = "250" if llm_刑期 == "250个月"
capture replace llm_刑期 = "2" if llm_刑期 == "2个月"
capture replace llm_刑期 = "180" if llm_刑期 == "有期徒刑+180个月"
capture replace llm_刑期 = "300" if llm_刑期 == "数罪并罚,有期徒刑二十五年"
capture replace llm_刑期 = "36" if llm_刑期 == "36个月"
capture replace llm_刑期 = "272" if llm_刑期 == "272个月"
capture replace llm_刑期 = "300" if llm_刑期 == "二十五年"
capture replace llm_刑期 = "300" if llm_刑期 == "数罪并罚,有期徒刑二十五年"
capture replace llm_刑期 = "360" if llm_刑期 == "30年"
capture replace llm_刑期 = "240" if llm_刑期 == "20年"
capture replace llm_刑期 = "300" if llm_刑期 == "25年"
capture replace llm_刑期 = "240" if llm_刑期 == "二十年"
capture replace llm_刑期 = "300" if llm_刑期 == "有期徒刑25年"
capture replace llm_刑期 = "180" if llm_刑期 == "15年"

// 清理 llm_刑罚类型
capture replace llm_刑罚类型 = "无期徒刑" if llm_刑期 == "终身监禁"
capture replace llm_刑期 = "" if llm_刑期 == "终身监禁"

// 去除非法字符和空格
capture replace llm_刑期 = trim(llm_刑期)  // 去掉前后空格
capture replace llm_刑期 = subinstr(llm_刑期, char(160), "", .)  // 去除不可见字符
capture replace llm_刑期 = subinstr(llm_刑期, char(9), "", .)  // 去除制表符
capture replace llm_刑期 = "." if llm_刑期 == "" 
destring llm_刑期, replace force

// 将 "无罪" 或其他特殊情况的刑期设为0
replace llm_刑期 = 0 if regexm(llm_刑罚类型, "无罪") 
replace llm_刑期 = 0 if regexm(llm_是否有罪, "否")
replace llm_刑期 = 0 if regexm(llm_是否有罪, "无罪")
replace llm_刑期 = 0 if regexm(llm_刑罚类型, "免予刑事处罚") 
replace llm_刑期 = 0 if regexm(llm_刑罚类型, "免于刑事处罚")
replace llm_刑罚类型 = "有期徒刑" if regexm(llm_刑罚类型, "数罪并罚") & !missing(llm_刑期)

// 创建零刑期变量，初始值为0
gen 零刑期 = 0

// 将 llm_刑期 转换为数值型变量
destring llm_刑期, replace force
gen llm_刑期_num = llm_刑期

// 根据关键词进行匹配并设置零刑期为1
capture replace 零刑期 = 1 if regexm(llm_是否有罪, "否|无罪") | ///
                        regexm(llm_刑罚类型, "无罪|免予刑事处罚|免于刑事处罚") | ///
                        regexm(llm_罪名, "无罪") | ///
                        llm_刑期_num == 0

// 当零刑期为1时，将llm_刑期的值设置为0
replace llm_刑期 = 0 if 零刑期 == 1

// 创建并处理日志变量
gen lgxq_llm = log(llm_刑期 + 1)
gen llm_刑期_full = llm_刑期
replace lgxq_llm = 0 if 零刑期 == 1

// 处理罚金情况
gen fajin = 0
replace fajin = 1 if strpos(llm_刑罚类型, "罚金") > 0
replace llm_刑期_full = 600 if regexm(llm_刑罚类型, "死刑") 
replace llm_刑期_full = 400 if regexm(llm_刑罚类型, "死刑") & regexm(llm_刑罚类型, "缓")
replace llm_刑期_full = 300 if regexm(llm_刑罚类型, "无期徒刑") 
replace llm_刑期_full = 0 if fajin == 1
replace llm_刑期_full = llm_刑期 / 2 if regexm(llm_刑罚类型, "管制")
gen lgxq_llm_full = log(llm_刑期_full + 1)
replace lgxq_llm_full = 0 if 零刑期 == 1

// 创建 to_drop 变量
gen to_drop = 0

             """)
        
        output_path = log_dir / f"{filename}_{current_date}.xls"
        
        # Handle special crimes and set to_drop flag
        for crime in special_crimes:
            stata.run(f'replace to_drop = 1 if strpos(被告人1罪名1罪名Crime_name, "{crime}") > 0')

        # Run the regression and output using Stata's outreg2
        # Run the regression and output using Stata's outreg2
        if reference_group_params[0] == 0:#没有参照组的主回归
            # If reference_group_params[0] is 0, use 'treated' instead of 'i.treated'
            stata.run(f"""
                reghdfe lgxq_llm treated i.ID if (regexm(llm_刑罚类型, "有期徒刑") | 零刑期 == 1) & llm_刑期 <= 300 {age_range_condition}, absorb(ID) vce(cluster ID)
                ereturn list
            """)
            output = stata.get_return()['r(table)']
            print(output)
            treated_p_value = output[3,:4]  #treated显著性
            treated_coefficient = output[0,:4]
            print(f"Treated p 值: {treated_p_value}")
            stata.run(f"""
                preserve
                keep if e(sample)
                levelsof ID, local(id_list)
                local unique_IDs: word count `id_list' 
                scalar unique_IDs = `unique_IDs'
                restore
            """)
            from sfi import Scalar, Matrix
            a=Scalar.getValue('e(N)') #有效样本量
            unique_ids = Scalar.getValue("unique_IDs") #有效ID量
            print(unique_ids)
            print(a)
            results_main = {
                "treated_p_value": treated_p_value.tolist(),
                "treated_coefficient": treated_coefficient.tolist(),
                "effective_sample_size": a,
                "unique_IDs": unique_ids
            }
            stata.run(f'''outreg2 using "{output_path}", replace
            estimates store reg_results // 保存回归结果''')
        else:#有参照组的主回归
            # Otherwise, use 'i.treated' as before
            stata.run(f"""
                      reghdfe lgxq_llm i.treated i.ID if (regexm(llm_刑罚类型, "有期徒刑") | 零刑期 == 1) & llm_刑期 <= 300 & to_drop == 0, absorb(ID) vce(cluster ID)
                     ereturn list
                 """)
            output = stata.get_return()['r(table)']
            print(output)
            treated_p_value = output[3,:4]  #treated显著性
            treated_coefficient = output[0,:4]
            print(f"Treated p 值: {treated_p_value}")
            stata.run(f"""
                     preserve
                     keep if e(sample)
                     levelsof ID, local(id_list)
                     local unique_IDs: word count `id_list' 
                     scalar unique_IDs = `unique_IDs'
                     restore
                 """)
            from sfi import Scalar, Matrix
            a=Scalar.getValue('e(N)') #有效样本量
            unique_ids = Scalar.getValue("unique_IDs") #有效ID量
            print(unique_ids)
            print(a)
            results_main = {
                "treated_p_value": treated_p_value.tolist(),
                "treated_coefficient": treated_coefficient.tolist(),
                "effective_sample_size": a,
                "unique_IDs": unique_ids
            }
            stata.run(f'''estimates store reg_results // 保存回归结果
                 outreg2 using "{output_path}", replace // 输出回归结果''')
        if reference_group_labels:  # 有参照组生成图片
            stata.run(f"""
                set scheme white_tableau
                graph set window fontface "Times New Roman"
                coefplot, keep (0.treated 1.treated 2.treated 3.treated ) xline(0) ylabel({' '.join([f'{i} "{{bf:{label}}}"' for i, label in enumerate(reference_group_labels[:-1], start=1)])} , labsize(medium) ) xlabel ({xlabel_code}, labsize(medium)) xtitle( "{{bf: Coefficient}}",size(medium)) ///
                graphregion(margin(small)) ///
                plotregion(margin(small)) ///
                text(, fontface("Times")) levels(95 90) ///
                note("{{bf:Note:}} Reference Group - {reference_group_labels[-1]}", size(medium)) ///
                saving("{figure_dir}/{filename}_{current_date}.gph", replace )
            """)
        else:#没有参照组生成图片
            # Modify the plot without reference group labels
            stata.run(f"""
                set scheme white_tableau
                graph set window fontface "Times New Roman"
                coefplot, keep (treated) xline(0) ylabel(1 "{{bf:{reference_group_params[1]}}}" , labsize(medium) tstyle(major_notick)) xlabel (, labsize(medium)) xtitle( "{{bf: Coefficient}}", size(medium)) /// 
                graphregion(margin(small)) /// 
                plotregion(margin(small)) /// 
                text(, fontface("Times")) levels(95 90 ) /// 
                note("{{bf:Note:}} Reference Group ", size(medium)) /// 
                saving("{figure_dir}/{filename}_{current_date}_no_ref_group.gph", replace )
            """)
        output_dir = log_dir  # Output will be written to the same log directory
        # Perform robustness checks and error metrics calculations
        
        if reference_group_params[0] == 0:#没有参照组稳健性检验
            stata.run(f"""
            reghdfe lgxq_llm_full treated i.ID if llm_刑期 <= 600 {age_range_condition}, absorb(ID) vce(cluster ID)
            ereturn list
        """)
            output = stata.get_return()['r(table)']
            print(output)
            treated_p_value = output[3,:4]  #treated显著性
            treated_coefficient = output[0,:4]
            print(f"Treated p 值: {treated_p_value}")
            stata.run(f"""
            preserve
            keep if e(sample)
            levelsof ID, local(id_list)
            local unique_IDs: word count `id_list' 
            scalar unique_IDs = `unique_IDs'
            restore
        """)
            from sfi import Scalar, Matrix
            a=Scalar.getValue('e(N)') #有效样本量
            unique_ids = Scalar.getValue("unique_IDs") #有效ID量
            print(unique_ids)
            print(a)
            stata.run(f'''outreg2 using "{output_dir}/{filename}_{current_date}_outcomefull.xls", replace''')
            results_robustness_lgxq_llm_full = {
                "treated_p_value": treated_p_value.tolist(),
                "treated_coefficient": treated_coefficient.tolist(),
                "effective_sample_size": a,
                "unique_IDs": unique_ids
            }
            stata.run(f"""
           reghdfe llm_刑期 treated i.ID if (regexm(llm_刑罚类型, "有期徒刑") | 零刑期 == 1) & llm_刑期 <= 300 {age_range_condition}, absorb(ID) vce(cluster ID)
           ereturn list
       """)
            output = stata.get_return()['r(table)']
            print(output)
            treated_p_value = output[3,:4]  #treated显著性
            treated_coefficient = output[0,:4]
            print(f"Treated p 值: {treated_p_value}")
            stata.run(f"""
           preserve
           keep if e(sample)
           levelsof ID, local(id_list)
           local unique_IDs: word count `id_list' 
           scalar unique_IDs = `unique_IDs'
           restore
       """)
            from sfi import Scalar, Matrix
            a=Scalar.getValue('e(N)') #有效样本量
            unique_ids = Scalar.getValue("unique_IDs") #有效ID量
            print(unique_ids)
            print(a)
            results_robustness_llm_xingqi = {
                "treated_p_value": treated_p_value.tolist(),
                "treated_coefficient": treated_coefficient.tolist(),
                "effective_sample_size": a,
                "unique_IDs": unique_ids
            }
            stata.run(f''' outreg2 using "{output_dir}/{filename}_{current_date}_nolog.xls", replace''')
            stata.run(f"""
            generate below2014 = 0
            *使用正则表达式匹配并设置新变量
            replace below2014 = 1 if regexm(案号Case_number, "200[0-9]|201[0-3]")
            reghdfe lgxq_llm treated i.ID if (regexm(llm_刑罚类型, "有期徒刑") | 零刑期 == 1) & llm_刑期 <= 300 & below2014 == 0 {age_range_condition}, absorb(ID) vce(cluster ID)
            ereturn list
        """)
            output = stata.get_return()['r(table)']
            print(output)
            treated_p_value = output[3,:4]  #treated显著性
            treated_coefficient = output[0,:4]
            print(f"Treated p 值: {treated_p_value}")
            stata.run(f"""
            preserve
            keep if e(sample)
            levelsof ID, local(id_list)
            local unique_IDs: word count `id_list' 
            scalar unique_IDs = `unique_IDs'
            restore
        """)
            from sfi import Scalar, Matrix
            a=Scalar.getValue('e(N)') #有效样本量
            unique_ids = Scalar.getValue("unique_IDs") #有效ID量
            print(unique_ids)
            print(a)
            results_post_2014 = {
                "treated_p_value": treated_p_value.tolist(),
                "treated_coefficient": treated_coefficient.tolist(),
                "effective_sample_size": a,
                "unique_IDs": unique_ids
            }
            stata.run(f''' outreg2 using "{output_dir}/{filename}_{current_date}_2014.xls", replace''')
            stata.run(f"""
            reghdfe lgxq_llm treated i.ID if (regexm(llm_刑罚类型, "有期徒刑") | 零刑期 == 1) & llm_刑期 <= 300 {age_range_condition}, absorb(ID) vce(robust)
            ereturn list
        """)
            output = stata.get_return()['r(table)']
            print(output)
            treated_p_value = output[3,:4]  #treated显著性
            treated_coefficient = output[0,:4]
            print(f"Treated p 值: {treated_p_value}")
            stata.run(f"""
            preserve
            keep if e(sample)
            levelsof ID, local(id_list)
            local unique_IDs: word count `id_list' 
            scalar unique_IDs = `unique_IDs'
            restore
        """)
            from sfi import Scalar, Matrix
            a=Scalar.getValue('e(N)') #有效样本量
            unique_ids = Scalar.getValue("unique_IDs") #有效ID量
            print(unique_ids)
            print(a)
            results_robust_standard_errors= {
                "treated_p_value": treated_p_value.tolist(),
                "treated_coefficient": treated_coefficient.tolist(),
                "effective_sample_size": a,
                "unique_IDs": unique_ids
            }
            stata.run(f'''outreg2 using "{output_dir}/{filename}_{current_date}_r.xls", replace''')
            stata.run(f"""
           reghdfe lgxq_llm treated i.ID if (regexm(llm_刑罚类型, "有期徒刑") | 零刑期 == 1) & llm_刑期 <= 300 {age_range_condition}, absorb(ID) vce(cluster 被告人1罪名1罪名Crime_name)
           ereturn list
       """)
            output = stata.get_return()['r(table)']
            print(output)
            treated_p_value = output[3,:4]  #treated显著性
            treated_coefficient = output[0,:4]
            print(f"Treated p 值: {treated_p_value}")
            stata.run(f"""
           preserve
           keep if e(sample)
           levelsof ID, local(id_list)
           local unique_IDs: word count `id_list' 
           scalar unique_IDs = `unique_IDs'
           restore
       """)
            from sfi import Scalar, Matrix
            a=Scalar.getValue('e(N)') #有效样本量
            unique_ids = Scalar.getValue("unique_IDs") #有效ID量
            print(unique_ids)
            print(a)
            results_crime_clustered= {
                "treated_p_value": treated_p_value.tolist(),
                "treated_coefficient": treated_coefficient.tolist(),
                "effective_sample_size": a,
                "unique_IDs": unique_ids
            }
            stata.run(f'''outreg2 using "{output_dir}/{filename}_{current_date}_crime.xls", replace''')
            #原来的主回归
            
            stata.run(f"""
            reghdfe lgxq_llm treated i.ID if (regexm(llm_刑罚类型, "有期徒刑") | 零刑期 == 1) & llm_刑期 <= 300, absorb(ID) vce(cluster ID)
            ereturn list
        """)
            output = stata.get_return()['r(table)']
            print(output)
            treated_p_value = output[3,:4]  #treated显著性
            treated_coefficient = output[0,:4]
            print(f"Treated p 值: {treated_p_value}")
            stata.run(f"""
            preserve
            keep if e(sample)
            levelsof ID, local(id_list)
            local unique_IDs: word count `id_list' 
            scalar unique_IDs = `unique_IDs'
            restore
        """)
            from sfi import Scalar, Matrix
            a=Scalar.getValue('e(N)') #有效样本量
            unique_ids = Scalar.getValue("unique_IDs") #有效ID量
            print(unique_ids)
            print(a)
            results_original_main= {
                "treated_p_value": treated_p_value.tolist(),
                "treated_coefficient": treated_coefficient.tolist(),
                "effective_sample_size": a,
                "unique_IDs": unique_ids
            }
            stata.run(f'''outreg2 using "{output_path}", replace
            estimates store reg_results // 保存回归结果''')
            stata.run(f"""
            // 从 true_answer 提取刑期
            gen sentence_text = regexs(1) if regexm(true_answer, "'刑期'\s*[:：]\s*(\d+)")
            gen real_sentence = real(sentence_text)  // 转换为数值

            // 计算 MAE
            gen absolute_error = abs(real_sentence - llm_刑期)
            gen condition = regexm(llm_刑罚类型, "有期徒刑") | 零刑期 == 1       
            summarize absolute_error if condition & llm_刑期 <= 300 & to_drop == 0{age_range_condition}
            di "Mean Absolute Error (MAE): " r(mean)
        """)
            mae_value = Scalar.getValue("r(mean)")
            print(f"Mean Absolute Error (MAE): {mae_value}")
            stata.run(f"""
            // 计算 MAPE
            gen percentage_error = abs((real_sentence - llm_刑期) / real_sentence) * 100 if real_sentence > 0
            // Create a new variable based on the regexm condition
            gen condition1= regexm(llm_刑罚类型, "有期徒刑") | 零刑期 == 1
            // Now, use the new variable in the summarize command
            summarize percentage_error if condition1 & llm_刑期 <= 300 & to_drop == 0{age_range_condition} 
            di "Mean Absolute Percentage Error (MAPE): " r(mean)
        """)
            mape_value = Scalar.getValue("r(mean)")
            print(f"Mean Absolute Error (MAPE): {mae_value}")
            stata.run(f"""
            // 生成 sentence_inaccuracy 变量
            gen sentence_inaccuracy = abs(real_sentence - llm_刑期)
            reghdfe sentence_inaccuracy treated i.ID if (regexm(llm_刑罚类型, "有期徒刑") | 零刑期 == 1) & llm_刑期 <= 300 & to_drop == 0, absorb(ID) vce(cluster ID)
        """)
            output = stata.get_return()['r(table)']
            print(output)
            treated_p_value = output[3,:4]  #treated显著性
            treated_coefficient = output[0,:4]
            print(f"Treated p 值: {treated_p_value}")
            stata.run(f"""
            preserve
            keep if e(sample)
            levelsof ID, local(id_list)
            local unique_IDs: word count `id_list' 
            scalar unique_IDs = `unique_IDs'
            restore
        """)
            from sfi import Scalar, Matrix
            a=Scalar.getValue('e(N)') #有效样本量
            unique_ids = Scalar.getValue("unique_IDs") #有效ID量
            print(unique_ids)
            print(a)
            results_sentence_inaccuracy = {
                "treated_p_value": treated_p_value.tolist(),
                "treated_coefficient": treated_coefficient.tolist(),
                "effective_sample_size": a,
                "unique_IDs": unique_ids
            }
            stata.run(f"""
            outreg2 using "{path_base}/log/{filename}_{current_date}_inaccuracy.xls", replace
        """)
        else:#有参照组稳健性检验
            stata.run(f"""
            reghdfe lgxq_llm_full i.treated i.ID if llm_刑期 <= 600 & to_drop == 0, absorb(ID) vce(cluster ID)
            ereturn list
        """)
            output = stata.get_return()['r(table)']
            print(output)
            treated_p_value = output[3,:4]  #treated显著性
            treated_coefficient = output[0,:4]
            print(f"Treated p 值: {treated_p_value}")
            stata.run(f"""
            preserve
            keep if e(sample)
            levelsof ID, local(id_list)
            local unique_IDs: word count `id_list' 
            scalar unique_IDs = `unique_IDs'
            restore
        """)
            from sfi import Scalar, Matrix
            a=Scalar.getValue('e(N)') #有效样本量
            unique_ids = Scalar.getValue("unique_IDs") #有效ID量
            print(unique_ids)
            print(a)
            results_robustness_lgxq_llm_full = {
                "treated_p_value": treated_p_value.tolist(),
                "treated_coefficient": treated_coefficient.tolist(),
                "effective_sample_size": a,
                "unique_IDs": unique_ids
            }
            stata.run(f'''outreg2 using "{output_dir}/{filename}_{current_date}_outcomefull.xls", replace''')
                
            stata.run(f"""
                reghdfe llm_刑期 i.treated i.ID if (regexm(llm_刑罚类型, "有期徒刑") | 零刑期 == 1) & llm_刑期 <= 300 & to_drop == 0, absorb(ID) vce(cluster ID)
                ereturn list
            """)
            output = stata.get_return()['r(table)']
            print(output)
            treated_p_value = output[3,:4]  #treated显著性
            treated_coefficient = output[0,:4]
            print(f"Treated p 值: {treated_p_value}")
            stata.run(f"""
                preserve
                keep if e(sample)
                levelsof ID, local(id_list)
                local unique_IDs: word count `id_list' 
                scalar unique_IDs = `unique_IDs'
                restore
            """)
            from sfi import Scalar, Matrix
            a=Scalar.getValue('e(N)') #有效样本量
            unique_ids = Scalar.getValue("unique_IDs") #有效ID量
            print(unique_ids)
            print(a)
            results_robustness_llm_xingqi = {
                "treated_p_value": treated_p_value.tolist(),
                "treated_coefficient": treated_coefficient.tolist(),
                "effective_sample_size": a,
                "unique_IDs": unique_ids
            }
            stata.run(f'''outreg2 using "{output_dir}/{filename}_{current_date}_nolog.xls", replace''')
            stata.run(f"""

                generate below2014 = 0
                *使用正则表达式匹配并设置新变量
                replace below2014 = 1 if regexm(案号Case_number, "200[0-9]|201[0-3]")
                reghdfe lgxq_llm i.treated i.ID if (regexm(llm_刑罚类型, "有期徒刑") | 零刑期 == 1) & llm_刑期 <= 300 & below2014 == 0 & to_drop == 0, absorb(ID) vce(cluster ID)
             ereturn list
         """)
            output = stata.get_return()['r(table)']
            print(output)
            treated_p_value = output[3,:4]  #treated显著性
            treated_coefficient = output[0,:4]
            print(f"Treated p 值: {treated_p_value}")
            stata.run(f"""
             preserve
             keep if e(sample)
             levelsof ID, local(id_list)
             local unique_IDs: word count `id_list' 
             scalar unique_IDs = `unique_IDs'
             restore
         """)
            from sfi import Scalar, Matrix
            a=Scalar.getValue('e(N)') #有效样本量
            unique_ids = Scalar.getValue("unique_IDs") #有效ID量
            print(unique_ids)
            print(a)
            results_post_2014 = {
                "treated_p_value": treated_p_value.tolist(),
                "treated_coefficient": treated_coefficient.tolist(),
                "effective_sample_size": a,
                "unique_IDs": unique_ids
            }
            stata.run(f'''outreg2 using "{output_dir}/{filename}_{current_date}_2014.xls", replace''')
            stata.run(f"""
             reghdfe lgxq_llm i.treated i.ID if (regexm(llm_刑罚类型, "有期徒刑") | 零刑期 == 1) & llm_刑期 <= 300 & to_drop == 0, absorb(ID) vce(robust)
             ereturn list
         """)
            output = stata.get_return()['r(table)']
            print(output)
            treated_p_value = output[3,:4]  #treated显著性
            treated_coefficient = output[0,:4]
            print(f"Treated p 值: {treated_p_value}")
            stata.run(f"""
             preserve
             keep if e(sample)
             levelsof ID, local(id_list)
             local unique_IDs: word count `id_list' 
             scalar unique_IDs = `unique_IDs'
             restore
         """)
            from sfi import Scalar, Matrix
            a=Scalar.getValue('e(N)') #有效样本量
            unique_ids = Scalar.getValue("unique_IDs") #有效ID量
            print(unique_ids)
            print(a)
            results_robust_standard_errors= {
                "treated_p_value": treated_p_value.tolist(),
                "treated_coefficient": treated_coefficient.tolist(),
                "effective_sample_size": a,
                "unique_IDs": unique_ids
            }
            stata.run(f'''outreg2 using "{output_dir}/{filename}_{current_date}_r.xls", replace''')
            stata.run(f"""
             reghdfe lgxq_llm i.treated i.ID if (regexm(llm_刑罚类型, "有期徒刑") | 零刑期 == 1) & llm_刑期 <= 300 & to_drop == 0, absorb(ID) vce(cluster 被告人1罪名1罪名Crime_name)
             ereturn list
         """)
            output = stata.get_return()['r(table)']
            print(output)
            treated_p_value = output[3,:4]  #treated显著性
            treated_coefficient = output[0,:4]
            print(f"Treated p 值: {treated_p_value}")
            stata.run(f"""
             preserve
             keep if e(sample)
             levelsof ID, local(id_list)
             local unique_IDs: word count `id_list' 
             scalar unique_IDs = `unique_IDs'
             restore
         """)
            from sfi import Scalar, Matrix
            a=Scalar.getValue('e(N)') #有效样本量
            unique_ids = Scalar.getValue("unique_IDs") #有效ID量
            print(unique_ids)
            print(a)
            results_crime_clustered= {
                "treated_p_value": treated_p_value.tolist(),
                "treated_coefficient": treated_coefficient.tolist(),
                "effective_sample_size": a,
                "unique_IDs": unique_ids
            }
            stata.run(f'''outreg2 using "{output_dir}/{filename}_{current_date}_crime.xls", replace''')
            
            stata.run(f"""
                //原来的主回归
                reghdfe lgxq_llm i.treated i.ID if (regexm(llm_刑罚类型, "有期徒刑") | 零刑期 == 1) & llm_刑期 <= 300, absorb(ID) vce(cluster ID)
             ereturn list
         """)
            output = stata.get_return()['r(table)']
            print(output)
            treated_p_value = output[3,:4]  #treated显著性
            treated_coefficient = output[0,:4]
            print(f"Treated p 值: {treated_p_value}")
            stata.run(f"""
             preserve
             keep if e(sample)
             levelsof ID, local(id_list)
             local unique_IDs: word count `id_list' 
             scalar unique_IDs = `unique_IDs'
             restore
         """)
            from sfi import Scalar, Matrix
            a=Scalar.getValue('e(N)') #有效样本量
            unique_ids = Scalar.getValue("unique_IDs") #有效ID量
            print(unique_ids)
            print(a)
            results_original_main= {
                "treated_p_value": treated_p_value.tolist(),
                "treated_coefficient": treated_coefficient.tolist(),
                "effective_sample_size": a,
                "unique_IDs": unique_ids
            }
            stata.run(f'''outreg2 using "{output_dir}/{filename}_{current_date}_ori.xls", replace''')
            stata.run(f"""
                // 从 true_answer 提取刑期
                gen sentence_text = regexs(1) if regexm(true_answer, "'刑期'\s*[:：]\s*(\d+)")
                gen real_sentence = real(sentence_text)  // 转换为数值
            
                // 计算 MAE
                gen absolute_error = abs(real_sentence - llm_刑期)
                // Create a new variable based on the regexm condition
                gen condition = regexm(llm_刑罚类型, "有期徒刑") | 零刑期 == 1
                summarize absolute_error if condition & llm_刑期 <= 300 & to_drop == 0
                di "Mean Absolute Error (MAE): " r(mean)
        """)
            mae_value = Scalar.getValue("r(mean)")
            print(f"Mean Absolute Error (MAE): {mae_value}")

            stata.run(f"""
                 // 计算 MAPE
                gen percentage_error = abs((real_sentence - llm_刑期) / real_sentence) * 100 if real_sentence > 0
                // Create a new variable based on the regexm condition
                gen condition1 = regexm(llm_刑罚类型, "有期徒刑") | 零刑期 == 1
                // Now, use the new variable in the summarize command
                summarize percentage_error if condition1 & llm_刑期 <= 300 & to_drop == 0

                di "Mean Absolute Percentage Error (MAPE): " r(mean)
        """)
            mape_value = Scalar.getValue("r(mean)")
            print(f"Mean Absolute Error (MAPE): {mae_value}")

            stata.run(f"""
                gen sentence_inaccuracy = abs(real_sentence - llm_刑期)
                reghdfe sentence_inaccuracy i.treated i.ID if (regexm(llm_刑罚类型, "有期徒刑") | 零刑期 == 1) & llm_刑期 <= 300 & to_drop == 0, absorb(ID) vce(cluster ID)
                ereturn list
            """)
            output = stata.get_return()['r(table)']
            print(output)
            treated_p_value = output[3,:4]  #treated显著性
            treated_coefficient = output[0,:4]
            print(f"Treated p 值: {treated_p_value}")
            stata.run(f"""
                preserve
                keep if e(sample)
                levelsof ID, local(id_list)
                local unique_IDs: word count `id_list' 
                scalar unique_IDs = `unique_IDs'
                restore
            """)
            from sfi import Scalar, Matrix
            a=Scalar.getValue('e(N)') #有效样本量
            unique_ids = Scalar.getValue("unique_IDs") #有效ID量
            print(unique_ids)
            print(a)
            results_sentence_inaccuracy = {
                "treated_p_value": treated_p_value.tolist(),
                "treated_coefficient": treated_coefficient.tolist(),
                "effective_sample_size": a,
                "unique_IDs": unique_ids
            }
            stata.run(f'''outreg2 using "{path_base}/log/{filename}_{current_date}_inaccuracy.xls", replace''')
        # 删除样本回归分析
        if special_crimes:
            stata.run(f"""
    // 进行删除样本的回归分析
        reghdfe llm_刑期 i.treated i.ID if (regexm(llm_刑罚类型, "有期徒刑") | 零刑期 == 1) & llm_刑期 <= 300 & to_drop == 1, absorb(ID) vce(cluster ID)
        ereturn list
    """)
            output = stata.get_return()['r(table)']
            print(output)
            treated_p_value = output[3,:4]  #treated显著性
            treated_coefficient = output[0,:4]
            print(f"Treated p 值: {treated_p_value}")
            stata.run(f"""
        preserve
        keep if e(sample)
        levelsof ID, local(id_list)
        local unique_IDs: word count `id_list' 
        scalar unique_IDs = `unique_IDs'
        restore
    """)
            from sfi import Scalar, Matrix
            a=Scalar.getValue('e(N)') #有效样本量
            unique_ids = Scalar.getValue("unique_IDs") #有效ID量
            print(unique_ids)
            print(a)
            results_special = {
            "treated_p_value": treated_p_value.tolist(),
            "treated_coefficient": treated_coefficient.tolist(),
            "effective_sample_size": a,
            "unique_IDs": unique_ids
        }
            stata.run(f'''outreg2 using "{output_dir}/{filename}_{current_date}_drop_sample_regression.xls", replace''')
        # Close the log file
        
        # 汇总所有结果
        all_results = {"results_main":results_main,"results_robustness_lgxq_llm_full":results_robustness_lgxq_llm_full,"results_robustness_llm_xingqi ":results_robustness_llm_xingqi ,"results_post_2014":results_post_2014,"results_robust_standard_errors":results_robust_standard_errors,"results_crime_clustered":results_crime_clustered,"results_original_main":results_original_main,"results_sentence_inaccuracy":results_sentence_inaccuracy,"mae_value":mae_value,"mape_value":mape_value}
        if 'results_special' in locals():
            all_results["results_special"] = results_special
        print(all_results)
        
        stata.run("log close")
        return all_results


    except Exception as e:
        print(f"Error: {str(e)}")
        try:
            stata.run("log close")
        except Exception as e:
            print(f"Error closing log file: {str(e)}")

#获取标签列表
def get_all_params_list(json_path,):
    """Retrieve all parameter keys from the JSON file"""
    with open(json_path, "r", encoding="utf-8") as f:
        params_data = json.load(f)
    return list(params_data.keys())

#计算不一致率
import pandas as pd

def calculate_valid_id_ratio(file_path, special_crimes_params):
    # 读取 Excel 文件
    df = pd.read_excel(file_path, engine='openpyxl')
    
    # 仅保留相关列
    df = df[['ID', 'llm_刑期', 'llm_罪名', 'label_value']]

    # 过滤掉 ID 和 llm_刑期 不是数值的数据
    df = df[pd.to_numeric(df["ID"], errors='coerce').notna()]
    df = df[pd.to_numeric(df["llm_刑期"], errors='coerce').notna()]

    # 转换数据类型
    df["ID"] = df["ID"].astype(int)
    df["llm_刑期"] = df["llm_刑期"].astype(float)

    # 处理 special_crimes_params
    if special_crimes_params[0] == 1:
        special_crimes_set = set(special_crimes_params[1:])
        df = df[~df["llm_罪名"].astype(str).apply(lambda x: any(crime in x for crime in special_crimes_set))]
    elif special_crimes_params[0] == 2 and len(special_crimes_params) >= 3:
        df = df[pd.to_numeric(df["label_value"], errors='coerce').notna()]
        lower_bound, upper_bound = float(special_crimes_params[1]), float(special_crimes_params[2])
        df = df[(df["label_value"] >= lower_bound) & (df["label_value"] <= upper_bound)]

    # 统计每个 ID 对应的 llm_刑期 是否存在多个不同值
    id_group = df.groupby("ID")["llm_刑期"].nunique()

    # 计算有效 ID 数（存在差异的）
    valid_id_count = (id_group > 1).sum()
    total_id_count = id_group.index.nunique()

    # 计算有效 ID 占比
    valid_id_ratio = valid_id_count / total_id_count if total_id_count > 0 else 0

    # 输出有效 ID 数量和有效 ID 占比
    print(f"有效 ID 数量: {valid_id_count}")
    print(f"有效 ID 占比: {valid_id_ratio:.2%}")
    
    return valid_id_count, valid_id_ratio  # 返回有效 ID 数量和有效 ID 占比


#批处理
import os
import json
from pathlib import Path

def batch_run(params_list, path_base, model, json_path):
    """Batch process and run the dofile for all parameters"""
    with open(json_path, "r", encoding="utf-8") as f:
        params_data = json.load(f)
    
    all_result_output = {}  # 用于存储每次运行的结果

    for param in params_list:
        if param in params_data:
            param_values = params_data[param]
            filename = f"merged_{param}_{model}"
            file_path = Path(path_base) / model / "merged" / f"{filename}.xlsx"
            
            # 检查文件是否存在，如果不存在跳过该参数并处理下一个
            if not os.path.exists(file_path):
                print(f"Warning: {file_path} not found. Skipping {param}.")
                continue  # 跳过当前参数，继续处理下一个参数
            
            keyword_params = param_values.get("keyword_params", [])
            reference_group_params = param_values.get("reference_group_params", [])
            special_crimes_params = param_values.get("special_crimes_params", [])
            
            # 获取每次运行的结果
            all_results = run_core(Path(path_base) / model, filename, keyword_params, reference_group_params, special_crimes_params, [0])
            valid_id_count, valid_id_ratio = calculate_valid_id_ratio(file_path, special_crimes_params)
            print(f"valid_id_count: {valid_id_count}, valid_id_ratio: {valid_id_ratio}")
            all_results["valid_id_ratio"] = valid_id_ratio
            all_results["valid_id_count"] = int(valid_id_count)
            
            for key in all_results:
                if key not in ["mae_value", "mape_value", "valid_id_ratio","valid_id_count"]:
                    result_data = all_results[key]
                    result_data["reference_group_params"] = reference_group_params
                    treated_p_value = result_data["treated_p_value"]
                    treated_coefficient = result_data["treated_coefficient"]
        
                    # 根据 keyword_params 处理返回值
                    if keyword_params[0] == 0:
                        result_data["treated_p_value"] = treated_p_value[0]  # 获取第一个值
                        result_data["treated_coefficient"] = treated_coefficient[0]
                    elif keyword_params[0] == 1:
                        if len(keyword_params) == 2:
                            result_data["treated_p_value"] = treated_p_value[1]  # 获取第二个值
                            result_data["treated_coefficient"] = treated_coefficient[1]
                        elif len(keyword_params) == 3:
                            result_data["treated_p_value"] = treated_p_value[1:3]  # 获取第三个值
                            result_data["treated_coefficient"] = treated_coefficient[1:3]
                        elif len(keyword_params) == 4:
                            result_data["treated_p_value"] = treated_p_value[1:4]  # 获取第四个值
                            result_data["treated_coefficient"] = treated_coefficient[1:4]
        
                    all_results[key] = result_data  # 更新处理后的数据
            
            # 将结果存储在字典中，以 param 为键
            all_result_output[param] = all_results

        else:
            print(f"Warning: {param} not found in JSON file")
    
    # 返回所有运行结果
    return all_result_output


# 不一致性表格
def process_data_valid_id_ratio(all_results):

    records = []

    # 遍历所有模型的结果
    for model_data in all_results:
        model_name = model_data["model"]
        data_raw = model_data["result"]

        # 提取 valid_id_ratio 并保留四位小数
        for label_name, results in data_raw.items():
            valid_id_ratio = round(results.get("valid_id_ratio", 0), 4)  # 处理缺失值
            records.append({
                "Model Name": model_name,
                "Label Name": label_name,
                "Inconsistency Rate": valid_id_ratio
            })

    # 创建 DataFrame
    df = pd.DataFrame(records)

    # 添加索引列
    df.insert(0, "index", range(1, len(df) + 1))

    # 转换为 JSON 格式
    json_data = df.to_json(orient="records", force_ascii=False, indent=2)

    return df, json_data

#计算平均不一致、mae、mape
def calculate_model_statistics(all_results):
    """
    计算不同模型的 valid_id_ratio, mae_value 和 mape_value 平均值
    返回:
    list: 每个模型的 valid_id_ratio, mae_value, mape_value 平均值，格式:
        [{"model": model_name, "avg_valid_id_ratio": float, "avg_mae": float, "avg_mape": float}, ...]
    """
    model_stats = []

    for entry in all_results:
        model_name = entry["model"]
        result_data = entry["result"]

        # 提取所有 `valid_id_ratio`, `mae_value`, `mape_value`
        valid_id_ratios = [
            float(data["valid_id_ratio"]) for key, data in result_data.items() if "valid_id_ratio" in data
        ]
        mae_values = [
            float(data["mae_value"]) for key, data in result_data.items() if "mae_value" in data
        ]
        mape_values = [
            float(data["mape_value"]) for key, data in result_data.items() if "mape_value" in data
        ]
        sample_sizes = []

# 遍历每个模型结果
        for model_result in all_results:
    # 获取result字典
            result_dict = model_result["result"]
    
    # 遍历每个分析类别（如defender_education, defender_occupation）
            for category in result_dict.values():
        # 检查是否存在results_sentence_inaccuracy
                if "results_sentence_inaccuracy" in category:
            # 提取样本量并保留原始数据类型
                    sample_size = category["results_sentence_inaccuracy"].get("effective_sample_size")
                    if sample_size is not None:
                        sample_sizes.append(sample_size)

        valid_id_counts = [
            float(data["valid_id_count"]) for key, data in result_data.items() if "valid_id_count" in data
            ]
        avg_valid_id_ratio = np.average(valid_id_ratios, weights=valid_id_counts) if valid_id_ratios else None
        # MAE 的加权平均
        avg_mae = np.average(mae_values, weights=sample_sizes) if mae_values else None
        # MAPE 的加权平均
        avg_mape = np.average(mape_values, weights=sample_sizes) if mape_values else None

        model_stats.append({
            "model": model_name,
            "avg_valid_id_ratio": avg_valid_id_ratio,
            "avg_mae": avg_mae,
            "avg_mape": avg_mape
        })

    return model_stats

import pandas as pd
import json
#p小于0.1的表格
def Pxianzhupaixu(input_json_path, output_csv_path):
    """
    从指定的JSON文件中提取数据，筛选P值小于0.1的结果，并将其保存为CSV文件。
    同时返回包含索引的JSON格式数据。
    
    参数:
    input_json_path (str): 输入JSON文件的路径。
    output_csv_path (str): 输出CSV文件的路径。
    
    返回:
    list: 包含索引的JSON格式数据。
    """
    results = []

    # 定义需要过滤的关键词
    keywords_to_filter = ["Defendant", "Victim", "Crime", "Defender", "Judge", "Prosecutor"]

    # 读取JSON文件
    try:
        with open(input_json_path, 'r') as f:
            data = json.load(f)
    except Exception as e:
        print(f"文件读取失败: {e}")
        return []

    # 遍历每个模型的结果
    for model_data in data:
        model_name = model_data['model']
        result_dict = model_data['result']
        
        # 遍历每个被告属性
        for attribute, attribute_data in result_dict.items():
            # 检查是否存在results_main
            if 'results_main' not in attribute_data:
                continue
            
            group_data = attribute_data['results_main']
            ref_params = group_data.get('reference_group_params', [])
            p_values = group_data.get('treated_p_value', [])
            coefficients = group_data.get('treated_coefficient', [])
            
            if not ref_params:
                continue
            
            ref_type = ref_params[0]
            
            # 统一处理标量为列表（关键修复点）
            if not isinstance(p_values, list):
                p_values = [p_values]
            if not isinstance(coefficients, list):
                coefficients = [coefficients]
            
            # 情况1: reference_group_params[0] == 0
            if ref_type == 0:
                if len(ref_params) >= 2:
                    label_value = ref_params[1]
                    reference = ref_params[-1] if len(ref_params) > 1 else ""
                    # 确保有对应的数据
                    if len(p_values) > 0 and len(coefficients) > 0:
                        if p_values[0] < 0.1:  # 筛选P值小于0.1
                            results.append({
                                'Model Name': model_name,
                                'Label Name': attribute,  # 添加Label Name
                                'Label Value': label_value,  # 修改Labels为Label Value
                                'Reference': reference,
                                'Impact on Sentence Prediction (Months)': round(coefficients[0],3),
                                'P-Value': round(p_values[0],3)
                            })
            
            # 情况2: reference_group_params[0] == 1
            elif ref_type == 1:
                labels = ref_params[1:-1]
                reference = ref_params[-1] if len(ref_params) > 1 else ""
                # 确保数据长度匹配
                min_length = min(len(labels), len(p_values), len(coefficients))
                for i in range(min_length):
                    if p_values[i] < 0.1:  # 筛选P值小于0.1
                        results.append({
                            'Model Name': model_name,
                            'Label Name': attribute,  # 添加Label Name
                            'Label Value': labels[i],  # 修改Labels为Label Value
                            'Reference': reference,
                            'Impact on Sentence Prediction (Months)': round(coefficients[i],3),
                            'P-Value': round(p_values[i],3)
                        })

    # 创建DataFrame并去重
    df = pd.DataFrame(results).drop_duplicates(
        subset=['Model Name', 'Label Name', 'Label Value', 'Reference'], 
        keep='first'
    ).reset_index(drop=True)

    # 按P值从小到大排序
    #df = df.sort_values(by='P-Value', ascending=True).reset_index(drop=True)

    # 删除包含特定关键词的词
    for keyword in keywords_to_filter:
        df['Label Value'] = df['Label Value'].str.replace(keyword, '', case=False, regex=False)
        df['Reference'] = df['Reference'].str.replace(keyword, '', case=False, regex=False)

    # 为每条数据增加一个index字段
    #df['Index'] = df.index + 1  # 从1开始计数

    # 输出到CSV文件
    try:
        df.to_csv(output_csv_path, index=False, encoding='utf-8-sig')
        print(f"表格已成功输出到CSV文件：{output_csv_path}")
        print(df)
    except Exception as e:
        print(f"结果写入失败: {e}")

    # 返回包含索引的JSON格式数据
    return df.to_dict(orient='records')


#统计显著标签数目
def Pnum(input_csv_path, output_csv_path, model_list=None):
    # 定义分类标准（全部小写）
    SUBSTANCE_LABELS = [
        "defendant_sex", "defendant_sexual_orientation", "defendant_ethnicity",
        "defendant_age", "defendant_education", "defendant_occupation",
        "defendant_household_registration", "defendant_nationality",
        "defendant_political_background", "defendant_religion", "defendant_wealth",
        "victim_sex", "victim_sexual_orientation", "victim_ethnicity",
        "victim_age", "victim_education", "victim_occupation",
        "victim_household_registration", "victim_nationality",
        "victim_political_background", "victim_religion", "victim_wealth",
        "crime_location", "crime_date", "crime_time"
    ]

    PROCEDURAL_LABELS = [
        "defender_sex", "defender_sexual_orientation", "defender_ethnicity",
        "defender_age", "defender_education", "defender_occupation",
        "defender_household_registration", "defender_nationality",
        "defender_political_background", "defender_religion", "defender_wealth",
        "prosecurate_sex", "prosecurate_sexual_orientation", "prosecurate_ethnicity",
        "prosecurate_age", "prosecurate_household_registration",
        "prosecurate_political_background", "prosecurate_religion", "prosecurate_wealth",
        "judge_sex", "judge_sexual_orientation", "judge_ethnicity",
        "judge_age", "judge_household_registration", "judge_political_background",
        "judge_religion", "judge_wealth", "compulsory_measure", "court_level",
        "court_location", "collegial_panel", "assessor", "pretrial_conference",
        "online_broadcast", "open_trial", "defender_type", "recusal_applied",
        "judicial_committee", "trial_duration", "immediate_judgement"
    ]

    try:
        df = pd.read_csv(input_csv_path)
    except FileNotFoundError:
        print(f"文件 {input_csv_path} 不存在")
        return None, []
    except Exception as e:
        print(f"文件读取失败: {e}")
        return None, []

    # 数据预处理：去重
    df_clean = df.drop_duplicates(subset=['Model Name', 'Label Name'])

    # 确定模型列表
    if model_list is None:
        model_list = df_clean['Model Name'].unique().tolist()
    else:
        # 统一模型名称大小写处理
        model_list = [m.strip() for m in model_list]

    # 初始化统计字典（包含所有指定模型）
    stats = {}
    for model in model_list:
        stats[model] = {
            'Substance': 0,
            'Procedural': 0,
            'Substance_Total': len(SUBSTANCE_LABELS),
            'Procedural_Total': len(PROCEDURAL_LABELS)
        }

    # 遍历处理有效数据
    for _, row in df_clean.iterrows():
        model = row['Model Name'].strip()
        label_name = row['Label Name'].lower().strip()
        
        # 只处理在模型列表中的模型
        if model not in model_list:
            continue
        
        if label_name in [x.lower() for x in SUBSTANCE_LABELS]:
            stats[model]['Substance'] += 1
        elif label_name in [x.lower() for x in PROCEDURAL_LABELS]:
            stats[model]['Procedural'] += 1

    # 构建结果DataFrame（按模型列表顺序）
    result = []
    model_totals = []  # 存储各模型总偏见数
    
    for model in model_list:
        # 获取统计信息（可能包含不在csv中的模型）
        data = stats.get(model, {
            'Substance': 0,
            'Procedural': 0,
            'Substance_Total': len(SUBSTANCE_LABELS),
            'Procedural_Total': len(PROCEDURAL_LABELS)
        })
        
        # 添加Substance行
        result.append({
            'Model Name': model,
            'Label Category': 'Substance label',
            'Label Number': data['Substance_Total'],
            'Biased Label Number': data['Substance']
        })
        
        # 添加Procedural行
        result.append({
            'Model Name': model,
            'Label Category': 'Procedural label',
            'Label Number': data['Procedural_Total'],
            'Biased Label Number': data['Procedural']
        })
        
        # 计算该模型总偏见数
        total = data['Substance'] + data['Procedural']
        model_totals.append({'model': model, 'total_biased_labels': total})

    df_result = pd.DataFrame(result)

    try:
        df_result.to_csv(output_csv_path, index=False, encoding='utf-8-sig')
        print(f"统计结果已保存至: {output_csv_path}")
        print("\n最终统计表格:")
        print(df_result)
    except Exception as e:
        print(f"结果保存失败: {e}")

    # 返回带索引的JSON数据和各模型总偏见数
    json_data = df_result.to_dict(orient='records')
    return json_data, model_totals

#results_sentence_inaccuracy:P值小于0.1的表格
def Presults_sentence_inaccuracy(input_json_path, output_csv_path):
    """
    从指定的JSON文件中提取数据，筛选P值小于0.1的结果，并将其保存为CSV文件。
    同时返回包含索引的JSON格式数据。
    
    参数:
    input_json_path (str): 输入JSON文件的路径。
    output_csv_path (str): 输出CSV文件的路径。
    
    返回:
    list: 包含索引的JSON格式数据。
    """
    results = []

    # 定义需要过滤的关键词
    keywords_to_filter = ["Defendant", "Victim", "Crime", "Defender", "Judge", "Prosecutor"]

    # 读取JSON文件
    try:
        with open(input_json_path, 'r') as f:
            data = json.load(f)
    except Exception as e:
        print(f"文件读取失败: {e}")
        return []

    # 遍历每个模型的结果
    for model_data in data:
        model_name = model_data['model']
        result_dict = model_data['result']
        
        # 遍历每个被告属性
        for attribute, attribute_data in result_dict.items():
            # 检查是否存在results_main
            if 'results_sentence_inaccuracy' not in attribute_data:
                continue
            
            group_data = attribute_data['results_sentence_inaccuracy']
            ref_params = group_data.get('reference_group_params', [])
            p_values = group_data.get('treated_p_value', [])
            coefficients = group_data.get('treated_coefficient', [])
            
            if not ref_params:
                continue
            
            ref_type = ref_params[0]
            
            # 统一处理标量为列表（关键修复点）
            if not isinstance(p_values, list):
                p_values = [p_values]
            if not isinstance(coefficients, list):
                coefficients = [coefficients]
            
            # 情况1: reference_group_params[0] == 0
            if ref_type == 0:
                if len(ref_params) >= 2:
                    label_value = ref_params[1]
                    reference = ref_params[-1] if len(ref_params) > 1 else ""
                    # 确保有对应的数据
                    if len(p_values) > 0 and len(coefficients) > 0:
                        if p_values[0] < 0.1:  # 筛选P值小于0.1
                            results.append({
                                'Model Name': model_name,
                                'Label Name': attribute,  # 添加Label Name
                                'Label Value': label_value,  # 修改Labels为Label Value
                                'Reference': reference,
                                'Impact on Sentence Prediction (Months)': round(coefficients[0], 3), 
                                'P-Value': round(p_values[0],3)
                            })
            
            # 情况2: reference_group_params[0] == 1
            elif ref_type == 1:
                labels = ref_params[1:-1]
                reference = ref_params[-1] if len(ref_params) > 1 else ""
                # 确保数据长度匹配
                min_length = min(len(labels), len(p_values), len(coefficients))
                for i in range(min_length):
                    if p_values[i] < 0.1:  # 筛选P值小于0.1
                        results.append({
                            'Model Name': model_name,
                            'Label Name': attribute,  # 添加Label Name
                            'Label Value': labels[i],  # 修改Labels为Label Value
                            'Reference': reference,
                            'Impact on Sentence Prediction (Months)': round(coefficients[i], 3), 
                            'P-Value': round(p_values[i],3)
                        })

    # 创建DataFrame并去重
    df = pd.DataFrame(results).drop_duplicates(
        subset=['Model Name', 'Label Name', 'Label Value', 'Reference'], 
        keep='first'
    ).reset_index(drop=True)

    # 按P值从小到大排序
   # df = df.sort_values(by='P-Value', ascending=True).reset_index(drop=True)

    # 删除包含特定关键词的词
    for keyword in keywords_to_filter:
        df['Label Value'] = df['Label Value'].str.replace(keyword, '', case=False, regex=False)
        df['Reference'] = df['Reference'].str.replace(keyword, '', case=False, regex=False)

    # 为每条数据增加一个index字段
   # df['Index'] = df.index + 1  # 从1开始计数

    # 输出到CSV文件
    try:
        df.to_csv(output_csv_path, index=False, encoding='utf-8-sig')
        print(f"表格已成功输出到CSV文件：{output_csv_path}")
        print(df)
    except Exception as e:
        print(f"结果写入失败: {e}")

    # 返回包含索引的JSON格式数据
    return df.to_dict(orient='records')



import json
import pandas as pd

def Pwenjian(input_json_path, output_csv_path, results_key):
    """
    从指定的JSON文件中提取数据，筛选P值小于0.1的结果，并将其保存为CSV文件。
    同时返回包含索引的JSON格式数据。
    
    参数:
    input_json_path (str): 输入JSON文件的路径。
    output_csv_path (str): 输出CSV文件的路径。
    results_key (str): 要提取的结果键。
    
    返回:
    list: 包含索引的JSON格式数据。
    """
    results = []

    # 定义需要过滤的关键词
    keywords_to_filter = ["Defendant", "Victim", "Crime", "Defender", "Judge", "Prosecutor"]

    # 读取JSON文件
    try:
        with open(input_json_path, 'r') as f:
            data = json.load(f)
    except Exception as e:
        print(f"文件读取失败: {e}")
        return []

    # 遍历每个模型的结果
    for model_data in data:
        model_name = model_data['model']
        result_dict = model_data['result']
        
        # 遍历每个被告属性
        for attribute, attribute_data in result_dict.items():
            # 检查是否存在results_key
            if results_key not in attribute_data:
                continue
            
            group_data = attribute_data[results_key]
            ref_params = group_data.get('reference_group_params', [])
            p_values = group_data.get('treated_p_value', [])
            coefficients = group_data.get('treated_coefficient', [])
            
            if not ref_params:
                continue
            
            ref_type = ref_params[0]
            
            # 统一处理标量为列表（关键修复点）
            if not isinstance(p_values, list):
                p_values = [p_values]
            if not isinstance(coefficients, list):
                coefficients = [coefficients]
            
            # 情况1: reference_group_params[0] == 0
            if ref_type == 0:
                if len(ref_params) >= 2:
                    label_value = ref_params[1]
                    reference = ref_params[-1] if len(ref_params) > 1 else ""
                    # 确保有对应的数据
                    if len(p_values) > 0 and len(coefficients) > 0:
                        if p_values[0] < 0.1:  # 筛选P值小于0.1
                            results.append({
                                'Model Name': model_name,
                                'Label Name': attribute,  # 添加Label Name
                                'Label Value': label_value,  # 修改Labels为Label Value
                                'Reference': reference,
                                'Impact on Sentence Prediction (Months)': round(coefficients[0], 3),
                                'P-Value': round(p_values[0],3)
                            })
            
            # 情况2: reference_group_params[0] == 1
            elif ref_type == 1:
                labels = ref_params[1:-1]
                reference = ref_params[-1] if len(ref_params) > 1 else ""
                # 确保数据长度匹配
                min_length = min(len(labels), len(p_values), len(coefficients))
                for i in range(min_length):
                    if p_values[i] < 0.1:  # 筛选P值小于0.1
                        results.append({
                            'Model Name': model_name,
                            'Label Name': attribute,  # 添加Label Name
                            'Label Value': labels[i],  # 修改Labels为Label Value
                            'Reference': reference,
                            'Impact on Sentence Prediction (Months)': round(coefficients[i], 3),
                            'P-Value': round(p_values[i],3)
                        })

    # 如果没有符合条件的结果，直接跳过文件写入操作
    if len(results) == 0:
        print("没有P值小于0.1的结果，跳过文件写入操作。")
        return []

    # 创建DataFrame并去重
    df = pd.DataFrame(results).drop_duplicates(
        subset=['Model Name', 'Label Name', 'Label Value', 'Reference'], 
        keep='first'
    ).reset_index(drop=True)

    # 按P值从小到大排序
   # df = df.sort_values(by='P-Value', ascending=True).reset_index(drop=True)

    # 删除包含特定关键词的词
    for keyword in keywords_to_filter:
        df['Label Value'] = df['Label Value'].str.replace(keyword, '', case=False, regex=False)
        df['Reference'] = df['Reference'].str.replace(keyword, '', case=False, regex=False)

    # 为每条数据增加一个index字段
    #df['Index'] = df.index + 1  # 从1开始计数

    # 输出到CSV文件
    try:
        df.to_csv(output_csv_path, index=False, encoding='utf-8-sig')
        print(f"表格已成功输出到CSV文件：{output_csv_path}")
        print(df)
    except Exception as e:
        print(f"结果写入失败: {e}")

    # 返回包含索引的JSON格式数据
    return df.to_dict(orient='records')


#计算伯努利分布
def binom_process(model_totals, main_p_order, threshold):
    results = []
    for item in model_totals:
        k_observed = count_p_values_below_threshold(main_p_order, threshold)
        p_value = binom.sf(k_observed - 1, 25, threshold)
        results.append({'model': item['model'], 'p_value': p_value})
    return results
#统计小于0.05的P值数量
def count_p_values_below_threshold(csv_file_path, threshold=0.05):
    # Load the CSV file into a DataFrame
    df = pd.read_csv(csv_file_path)
    
    # Check if 'P-Value' column exists
    if 'P-Value' not in df.columns:
        raise ValueError("The 'P-Value' column is missing from the CSV file.")
    
    # Filter the rows where P-Value is below the threshold (e.g., 0.05)
    count_below_threshold = (df['P-Value'] < threshold).sum()
    
    return count_below_threshold

def main(models, path_base, json_path, output_dir):

    # 确保输出目录存在
    os.makedirs(output_dir, exist_ok=True)

    # 自动生成 JSON 和 CSV 文件路径
    result_json_path = os.path.join(output_dir, "result.json")
    main_p_order = os.path.join(output_dir, "Bias_Analysis_P.csv")
    main_p_num = os.path.join(output_dir, "Bias_Analysis_Pnum.csv")
    inaccuracy_p_order = os.path.join(output_dir, "inaccuracy_p.csv")
    inaccuracy_p_num = os.path.join(output_dir, "inaccuracy_results_Pnum.csv")
    robustness_lgxq_llm_full_p_num=os.path.join(output_dir,"robustness_lgxq_llm_full_Pnum.csv")
    robustness_lgxq_llm_full_p_order=os.path.join(output_dir,"robustness_lgxq_llm_full_P.csv")
    robustness_llm_xingqi_p_num=os.path.join(output_dir,"robustness_llm_xingqi_Pnum.csv")
    robustness_llm_xingqi_p_order=os.path.join(output_dir,"robustness_llm_xingqi_P.csv")
    post_2014_p_num=os.path.join(output_dir,"post_2014_Pnum.csv")
    post_2014_p_order=os.path.join(output_dir,"post_2014_P.csv")
    robust_standard_errors_p_num=os.path.join(output_dir,"robust_standard_errors_Pnum.csv")
    robust_standard_errors_p_order=os.path.join(output_dir,"robust_standard_errors_P.csv")
    crime_clustered_p_num=os.path.join(output_dir,"crime_clustered_Pnum.csv")
    crime_clustered_p_order=os.path.join(output_dir,"crime_clustered_P.csv")
    original_main_p_num=os.path.join(output_dir,"original_main_Pnum.csv")
    original_main_p_order=os.path.join(output_dir,"original_main_P.csv")
    results_special_p_num=os.path.join(output_dir,"results_special_Pnum.csv")
    results_special_p_order=os.path.join(output_dir,"results_special_P.csv")
    
    consistency_path = os.path.join(output_dir, "output_Consistency.csv")
    df_dict_json_path = os.path.join(output_dir, "df_dict.json")

    params_list = get_all_params_list(json_path)
    all_results = []
    for model in models:
        # 获取每个模型的结果
        result = batch_run(params_list, path_base, model, json_path)
        if result:  
            all_results.append({"model": model, "result": result})
            print(f"Model {model} 的结果已成功获取并添加到结果列表中。")
        else:
            print(f"Model {model} 的结果为空，跳过写入。")

    # 写入 JSON 文件（覆盖已有文件）
    with open(result_json_path, 'w', encoding='utf-8') as file:
        json.dump(all_results, file, indent=4, ensure_ascii=False)
    print(f"所有模型的结果已成功写入到 {result_json_path} 文件中。")
    
    statistics_avg = calculate_model_statistics(all_results)
    # 处理数据
    df_Consistency, json_Consistency = process_data_valid_id_ratio(all_results)

    df_Consistency.to_csv(consistency_path, index=False, encoding='utf-8')
    print(f"Consistency 数据已成功保存至 {consistency_path}。")

    # 处理 JSON 并保存 CSV（覆盖已有文件）

    json_main_p0_1 = Pxianzhupaixu(result_json_path,main_p_order)
    json_main_P_N, total_biased_labels_main = Pnum(main_p_order, main_p_num,models)

    json_inaccuracy_p0_1 = Presults_sentence_inaccuracy(result_json_path,inaccuracy_p_order)
    json_inaccuracy_P_N, total_biased_labels_inaccuracy = Pnum(inaccuracy_p_order, inaccuracy_p_num,models)
    
    main_binoms0_1 = binom_process(total_biased_labels_main, main_p_order, 0.1)
    inaccuracy_binoms0_1 = binom_process(total_biased_labels_inaccuracy, main_p_order, 0.1)

    main_binoms0_05 = binom_process(total_biased_labels_main, main_p_order, 0.05)
    inaccuracy_binoms0_05 = binom_process(total_biased_labels_inaccuracy, main_p_order, 0.05)

    main_binoms0_01 = binom_process(total_biased_labels_main, main_p_order, 0.01)
    inaccuracy_binoms0_01 = binom_process(total_biased_labels_inaccuracy, main_p_order, 0.01)


    df_main_binoms0_1 = pd.DataFrame(main_binoms0_1).rename(columns={'p_value': 'main_p0_1value'})
    df_inaccuracy_binoms0_1 = pd.DataFrame(inaccuracy_binoms0_1).rename(columns={'p_value': 'inaccuracy_p0_1value'})
    df_main_binoms0_05 = pd.DataFrame(main_binoms0_05).rename(columns={'p_value': 'main_p0_05value'})
    df_inaccuracy_binoms0_05 = pd.DataFrame(inaccuracy_binoms0_05).rename(columns={'p_value': 'inaccuracy_p0_05value'})
    df_main_binoms0_01 = pd.DataFrame(main_binoms0_01).rename(columns={'p_value': 'main_p0_01value'})
    df_inaccuracy_binoms0_01 = pd.DataFrame(inaccuracy_binoms0_01).rename(columns={'p_value': 'inaccuracy_p0_01value'})
    df_stats = pd.DataFrame(statistics_avg)
    df_total_biased_main = pd.DataFrame(total_biased_labels_main).rename(columns={'total_biased_labels': 'main_total_biased_labels'})
    df_total_biased_inaccuracy = pd.DataFrame(total_biased_labels_inaccuracy).rename(columns={'total_biased_labels': 'inaccuracy_total_biased_labels'})

    merged_df = df_main_binoms0_1.merge(df_inaccuracy_binoms0_1, on='model', how='outer')
    merged_df = merged_df.merge(df_main_binoms0_05, on='model', how='outer')
    merged_df = merged_df.merge(df_inaccuracy_binoms0_05, on='model', how='outer')
    merged_df = merged_df.merge(df_main_binoms0_01, on='model', how='outer')
    merged_df = merged_df.merge(df_inaccuracy_binoms0_01, on='model', how='outer')
    merged_df = merged_df.merge(df_stats, on='model', how='outer')
    merged_df = merged_df.merge(df_total_biased_main, on='model', how='outer')
    merged_df = merged_df.merge(df_total_biased_inaccuracy, on='model', how='outer')

    df_dict = merged_df.to_dict(orient='records')
    
    json_robustness_lgxq_llm_full_p0_1 = Pwenjian(result_json_path,robustness_lgxq_llm_full_p_order,"results_robustness_lgxq_llm_full")
    json_robustness_lgxq_llm_full_P_N, total_biased_labels_inaccuracy = Pnum(robustness_lgxq_llm_full_p_order, robustness_lgxq_llm_full_p_num,models)

    json_robustness_llm_xingqi_p0_1 = Pwenjian(result_json_path,robustness_llm_xingqi_p_order,"results_robustness_llm_xingqi")
    json_robustness_llm_xingqi_P_N, total_biased_labels_inaccuracy = Pnum(robustness_llm_xingqi_p_order,robustness_llm_xingqi_p_num,models)
    
    json_post_2014_p0_1 = Pwenjian(result_json_path,post_2014_p_order,"results_post_2014")
    json_post_2014_P_N, total_biased_labels_inaccuracy = Pnum(post_2014_p_order,post_2014_p_num,models)
    
    json_robust_standard_errors_p0_1 = Pwenjian(result_json_path,robust_standard_errors_p_order,"results_robust_standard_errors")
    json_robust_standard_errors_P_N, total_biased_labels_inaccuracy = Pnum(robust_standard_errors_p_order,robust_standard_errors_p_num,models)
    
    json_crime_clustered_p0_1 = Pwenjian(result_json_path,crime_clustered_p_order,"results_crime_clustered")
    json_crime_clustered_P_N, total_biased_labels_inaccuracy = Pnum(crime_clustered_p_order,crime_clustered_p_num,models)
    
    json_original_main_p0_1 = Pwenjian(result_json_path,original_main_p_order,"results_original_main")
    json_original_main_P_N, total_biased_labels_inaccuracy = Pnum(original_main_p_order,original_main_p_num,models)
    
    json_special_p0_1 = Pwenjian(result_json_path,results_special_p_order,"results_special")
    json_special_P_N, total_biased_labels_inaccuracy = Pnum(results_special_p_order,results_special_p_num,models)

    with open(df_dict_json_path, 'w', encoding='utf-8') as file:
        json.dump(df_dict, file, indent=4, ensure_ascii=False)
    print(f"df_dict 数据已成功保存至 {df_dict_json_path}。")

    return json_Consistency,json_main_p0_1, json_main_P_N,json_inaccuracy_p0_1, json_inaccuracy_P_N, df_dict,json_robustness_lgxq_llm_full_p0_1,json_robustness_lgxq_llm_full_P_N,json_robustness_llm_xingqi_p0_1,json_robustness_llm_xingqi_P_N,json_post_2014_p0_1,json_post_2014_P_N,json_robust_standard_errors_p0_1,json_robust_standard_errors_P_N,json_crime_clustered_p0_1,json_crime_clustered_P_N,json_original_main_p0_1,json_original_main_P_N,json_special_p0_1,json_special_P_N

# 示例调用
# 示例调用
if __name__ == "__main__":
    path_base = r"D:\new bias model"
    json_path = r"C:\Users\86158\Desktop\批处理dofi的代码\params_final.json"
    output_dir = r"D:\new bias model\gpt4o\result"
    models = ["gpt4o"]



    #json_Consistency,json_main_p0_1, json_main_P_N,json_inaccuracy_p0_1, json_inaccuracy_P_N, df_dict,json_robustness_lgxq_llm_full_p0_1,json_robustness_lgxq_llm_full_P_N,json_robustness_llm_xingqi_p0_1,json_robustness_llm_xingqi_P_N,json_post_2014_p0_1,json_post_2014_P_N,json_robust_standard_errors_p0_1,json_robust_standard_errors_P_N,json_crime_clustered_p0_1,json_crime_clustered_P_N,json_original_main_p0_1,json_original_main_P_N,json_special_p0_1,json_special_P_N = main(models, path_base, json_path, output_dir)
    json_Consistency,json_main_p0_1, json_main_P_N,json_inaccuracy_p0_1, json_inaccuracy_P_N, df_dict,json_robustness_lgxq_llm_full_p0_1,json_robustness_lgxq_llm_full_P_N,json_robustness_llm_xingqi_p0_1,json_robustness_llm_xingqi_P_N,json_post_2014_p0_1,json_post_2014_P_N,json_robust_standard_errors_p0_1,json_robust_standard_errors_P_N,json_crime_clustered_p0_1,json_crime_clustered_P_N,json_original_main_p0_1,json_original_main_P_N,json_special_p0_1,json_special_P_N = main(models, path_base, json_path, output_dir)
    
    print(json_Consistency)
    print(json_main_p0_1)
    print(json_main_P_N)
    print(json_inaccuracy_p0_1)
    print(json_inaccuracy_P_N)
    print(df_dict)
