#!/usr/bin/env python3 """ Export Control DID Test -- Paper 6, Section 7 ============================================= US export controls on China (BIS Entity List expansion, Oct 7 2022) constrained Chinese firms' access to frontier GPU hardware. Tests whether constrained firms redirected toward parameter-efficient models. Subtests: T1 (DID): edge_share ~ post + constrained + post*constrained T2 (post-only size): log(param_b) ~ constrained + quarter_FE T3 (post-only edge 3B): chi-squared test of edge share at paper's <=3B threshold Migrated from scripts/test_export_control_did.py Review Notes (2026-03-12): Theory match: PARTIAL — PAPER ALREADY CORRECTED. Paper 6, Section 7 predicts constraint-induced optimization toward edge architectures. No Lean theorem — qualitative incentive reasoning only. Paper line 1078 NOW ACKNOWLEDGES: "unconstrained developers actually release a higher share of small (<=3B) models (45% vs 25%)... The constraint-induced optimization signal is in architecture (MoE, distillation, active/total parameter ratio) rather than size. An earlier draft incorrectly reported size distributions as the primary evidence." Tests T2 and T3 test the OLD (size-based) claim → correctly INCONSISTENT. The CORRECTED claim (architectural) is not testable from this data: only 30 MoE models total, ~1% rate for both groups. I/O audit: CORRECT. Variables are reasonable proxies for size-based claim. Data-signal alignment: WEAK. CRITICAL: Pre-treatment Chinese sample = 69 models from 2 NLP utility labs (uer, shibing624). ALL major Chinese AI firms (Qwen, DeepSeek, BAAI, etc.) appear only POST-treatment. DID identification broken. Post-only shows Chinese models 1.55x LARGER (p<0.0001). Code quality: CLEAN after 2026-03-11 rewrite. Added _ALL_SUBTESTS 2026-03-12. Confidence in verdict: HIGH — INCONSISTENT is correct; paper acknowledges it. Note: Stale scorecard entry 'export_control_chinese_edge_increase' from pre-rewrite code; no longer produced. Will be cleaned on next scorecard regen. Date of review: 2026-03-12 """ import warnings import numpy as np import pandas as pd from scipy import stats from common.config import COMPILED_DIR, RESULTS_DIR from common.data import load_compiled from common.plotting import setup_figure, setup_subplots, save_figure from common.stats import verdict, log_result warnings.filterwarnings('ignore') # Treatment date TREATMENT_DATE = pd.Timestamp('2022-10-07') # Chinese AI firms (treated) _ALL_SUBTESTS = [ 'export_control_did_edge_share', 'export_control_post_size', 'export_control_edge_3b', ] CHINESE_FIRMS = ['Qwen', 'alibaba', 'deepseek', 'THUDM', 'BAAI', 'baichuan', 'internlm', 'Yi', 'chatglm', 'modelscope', 'openbmb', '01-ai'] # US/EU control firms CONTROL_FIRMS = ['meta-llama', 'google', 'microsoft', 'mistralai', 'openai', 'EleutherAI', 'bigscience', 'stabilityai'] def load_huggingface_data(): """Load HuggingFace model-level data from compiled directory.""" try: return load_compiled('export_control_model_level') except FileNotFoundError: return None _cache = {} _SKIP_DESC_DID = ( "Difference-in-differences: do export-constrained Chinese AI firms produce " "more edge-compatible models (<=7B params) after Oct 2022 BIS Entity List? " "OLS: edge_share ~ post + constrained + post*constrained with HC1 robust SEs " "on quarterly HuggingFace data. Data not available." ) def _prepare_data(df): """Prepare common data columns. Returns (df, firm_col) or (None, None).""" date_col = next((c for c in ['release_date', 'date', 'created'] if c in df.columns), None) if date_col is None: return None, None df['date'] = pd.to_datetime(df[date_col], errors='coerce') df = df.dropna(subset=['date']) firm_col = next((c for c in ['author', 'organization'] if c in df.columns), None) if firm_col is None: return None, None if 'constrained' not in df.columns: df['constrained'] = df[firm_col].apply( lambda x: 1 if any(cf.lower() in str(x).lower() for cf in CHINESE_FIRMS) else (0 if any(uf.lower() in str(x).lower() for uf in CONTROL_FIRMS) else np.nan)) df['constrained'] = pd.to_numeric(df['constrained'], errors='coerce') df = df.dropna(subset=['constrained']) if 'post' not in df.columns: df['post'] = (df['date'] >= TREATMENT_DATE).astype(int) df['post'] = pd.to_numeric(df['post'], errors='coerce') # Edge compatibility from pre-computed column or param size if 'edge_compatible' in df.columns: df['edge_compatible'] = df['edge_compatible'].map( {True: 1.0, False: 0.0, 'True': 1.0, 'False': 0.0} ).fillna(pd.to_numeric(df['edge_compatible'], errors='coerce')) else: param_col = next((c for c in ['params_b', 'param_b', 'parameters_billion'] if c in df.columns), None) if param_col: df['edge_compatible'] = df[param_col].apply( lambda x: 1.0 if pd.notna(x) and x <= 7.0 else (0.0 if pd.notna(x) else np.nan)) df['quarter'] = df['date'].dt.to_period('Q') return df, firm_col def _test_did(df, firm_col): """T1: DID on quarterly edge share (kept for continuity, known broken).""" work = df.dropna(subset=['edge_compatible']).copy() qtr = work.groupby([firm_col, 'quarter', 'constrained', 'post']).agg( edge_share=('edge_compatible', 'mean'), n_models=('edge_compatible', 'count') ).reset_index() qtr['interaction'] = qtr['post'] * qtr['constrained'] try: import statsmodels.api as sm X = sm.add_constant(qtr[['post', 'constrained', 'interaction']]) y = qtr['edge_share'] model = sm.OLS(y, X).fit(cov_type='HC1') delta = model.params['interaction'] p_delta = model.pvalues['interaction'] sign_ok = delta > 0 verd = verdict(p_delta, sign_ok, threshold=0.10) except Exception: delta, p_delta, verd = np.nan, np.nan, 'SKIPPED' qtr = None return delta, p_delta, verd, qtr def _test_post_only_size(df): """T2: Post-only OLS: log(param_b) ~ constrained + quarter_FE.""" post = df[(df['post'] == 1) & df['param_b'].notna()].copy() if len(post) < 20: return np.nan, np.nan, 'SKIPPED', {} post['log_param'] = np.log(post['param_b'].clip(lower=0.01)) post['q_str'] = post['quarter'].astype(str) cn = post[post['constrained'] == 1]['log_param'] ctrl = post[post['constrained'] == 0]['log_param'] n_cn, n_ctrl = len(cn), len(ctrl) try: import statsmodels.api as sm dummies = pd.get_dummies(post['q_str'], drop_first=True, dtype=float) X = sm.add_constant(pd.concat([post[['constrained']], dummies], axis=1)) y = post['log_param'] model = sm.OLS(y, X).fit(cov_type='HC1') beta = model.params['constrained'] p_val = model.pvalues['constrained'] except Exception: beta, p_val = np.nan, np.nan # Prediction: constrained firms produce SMALLER models (negative beta) sign_ok = beta < 0 verd = verdict(p_val, sign_ok, threshold=0.10) details = { 'beta': float(beta), 'size_ratio': float(np.exp(beta)), 'n_cn': n_cn, 'n_ctrl': n_ctrl, 'cn_median_b': float(post[post['constrained'] == 1]['param_b'].median()), 'ctrl_median_b': float(post[post['constrained'] == 0]['param_b'].median()), } _cache['post_size'] = { 'cn_params': cn.values, 'ctrl_params': ctrl.values, 'beta': beta, 'p': p_val, } return beta, p_val, verd, details def _test_post_only_edge_3b(df): """T3: Post-only edge share at paper's <=3B threshold.""" post = df[(df['post'] == 1) & df['param_b'].notna()].copy() if len(post) < 20: return np.nan, np.nan, 'SKIPPED', {} post['edge_3b'] = (post['param_b'] <= 3.0).astype(float) cn = post[post['constrained'] == 1] ctrl = post[post['constrained'] == 0] cn_share = cn['edge_3b'].mean() ctrl_share = ctrl['edge_3b'].mean() # Contingency table for chi-squared table = np.array([ [cn['edge_3b'].sum(), len(cn) - cn['edge_3b'].sum()], [ctrl['edge_3b'].sum(), len(ctrl) - ctrl['edge_3b'].sum()] ]) chi2, p_chi = stats.chi2_contingency(table)[:2] # Prediction: constrained firms produce MORE <=3B models sign_ok = cn_share > ctrl_share verd = verdict(p_chi, sign_ok, threshold=0.10) details = { 'cn_edge_3b': float(cn_share), 'ctrl_edge_3b': float(ctrl_share), 'diff': float(cn_share - ctrl_share), 'n_cn': len(cn), 'n_ctrl': len(ctrl), 'paper_claim': 'CN=0.47 vs Ctrl=0.25', 'data_shows': f'CN={cn_share:.3f} vs Ctrl={ctrl_share:.3f} (OPPOSITE)', } _cache['edge_3b'] = { 'cn_share': cn_share, 'ctrl_share': ctrl_share, 'chi2': chi2, 'p': p_chi, } return chi2, p_chi, verd, details def main(): results = [] df = load_huggingface_data() if df is None: return [log_result('export_control_did_edge_share', np.nan, np.nan, 'SKIPPED', details={'reason': 'HuggingFace data not found'}, description=_SKIP_DESC_DID)] df, firm_col = _prepare_data(df) if df is None: return [log_result('export_control_did_edge_share', np.nan, np.nan, 'SKIPPED', details={'reason': 'Missing required columns'}, description=_SKIP_DESC_DID)] # ---- T1: DID (kept for continuity; known broken) ---- delta, p_delta, verd_did, qtr = _test_did(df, firm_col) results.append(log_result( 'export_control_did_edge_share', delta, p_delta, verd_did, details={'n_obs': len(qtr) if qtr is not None else 0, 'identification': 'broken_compositional_change'}, description="Difference-in-differences: do export-constrained Chinese AI firms " "produce more edge-compatible models (<=7B params) after Oct 2022 BIS " "Entity List? OLS: edge_share ~ post + constrained + post*constrained " "with HC1 robust SEs on quarterly HuggingFace data. Prediction: " "positive delta (constrained firms shift more toward edge). " "KNOWN LIMITATION: DID identification is broken — pre-treatment " "Chinese sample is 69 models from 2 NLP utility labs (uer, shibing624); " "all major Chinese AI firms (Qwen, DeepSeek, BAAI) appear only " "post-treatment. Control group also increased edge share by similar " "magnitude (+0.132 vs +0.137), indicating an industry trend " "(Llama 2, Mistral-7B) not an export control effect.")) # ---- T2: Post-only model size with quarter FE ---- beta, p_size, verd_size, details_size = _test_post_only_size(df) results.append(log_result( 'export_control_post_size', beta, p_size, verd_size, details=details_size, description="Post-only size comparison: are export-constrained Chinese AI models " "smaller than US/EU models after controlling for release quarter? " "OLS: log(param_b) ~ constrained + quarter_FE with HC1 robust SEs " "on post-Oct-2022 HuggingFace models with known parameter counts. " "Prediction: negative beta (Chinese models smaller). " "RESULT: Chinese models are 1.55x LARGER (beta=+0.44, p<0.0001), " "OPPOSITE to prediction. US/EU firms (Meta, Microsoft, Google) " "aggressively release small variants; Chinese firms focus on " "large flagship models.")) # ---- T3: Post-only edge share at paper's <=3B threshold ---- chi2, p_edge, verd_edge, details_edge = _test_post_only_edge_3b(df) results.append(log_result( 'export_control_edge_3b', chi2, p_edge, verd_edge, details=details_edge, description="Post-only edge share at paper's <=3B threshold: tests the paper's " "specific claim that 'constrained developers release 47%% edge-compatible " "models (<=3B) vs 25%% for unconstrained' (Paper 6, Section 7). " "Chi-squared test on 2x2 contingency table of constrained/control x " "edge/non-edge at <=3B threshold. Prediction: CN edge share > US/EU. " "DATA SHOWS OPPOSITE: CN=25%% vs US/EU=45%%. US/EU firms produce more " "sub-3B models (Meta Llama-1B/3B, Microsoft Phi-1.3B/2.7B/3.8B, " "Google Gemma-2B).")) _cache['results'] = results _cache['qtr'] = qtr _cache['df'] = df _cache['firm_col'] = firm_col return results def figures(): """Generate export control DID figures.""" if not _cache: main() paths = [] df = _cache.get('df') qtr = _cache.get('qtr') if df is None or len(df) == 0: return paths # Figure 1: DID — edge share pre/post by group + quarterly trend if qtr is not None and len(qtr) > 0: fig, axes = setup_subplots(1, 2, width=14, height=6) # Panel A: Pre/post edge share by group ax = axes[0] for label, name, color in [(1, 'Constrained (CN)', '#e74c3c'), (0, 'Control (US/EU)', '#3498db')]: sub = qtr[qtr['constrained'] == label] if len(sub) == 0: continue pre_val = sub[sub['post'] == 0]['edge_share'].mean() post_val = sub[sub['post'] == 1]['edge_share'].mean() ax.plot([0, 1], [pre_val, post_val], 'o-', color=color, label=name, linewidth=2, markersize=8) ax.set_xticks([0, 1]) ax.set_xticklabels(['Pre-Treatment', 'Post-Treatment']) ax.set_ylabel('Edge-Compatible Share (<=7B)') ax.set_title('DID: Both Groups Shift Similarly (delta~0)') ax.legend() ax.annotate('Industry trend, not export control effect', xy=(0.5, 0.02), xycoords='axes fraction', fontsize=8, fontstyle='italic', color='gray', ha='center') # Panel B: Quarterly edge share ax = axes[1] for label, name, color in [(1, 'Constrained', '#e74c3c'), (0, 'Control', '#3498db')]: sub = qtr[qtr['constrained'] == label].copy() sub['q_str'] = sub['quarter'].astype(str) grp = sub.groupby('q_str')['edge_share'].mean().sort_index() ax.plot(range(len(grp)), grp.values, 'o-', color=color, label=name, markersize=4) ax.axvline(x=2.5, color='gray', linestyle='--', alpha=0.7, label='Oct 2022 cutoff') ax.set_xlabel('Quarter') ax.set_ylabel('Mean Edge-Compatible Share') ax.set_title('Quarterly Edge Model Share by Group') ax.legend(fontsize=8) paths += save_figure(fig, 'export_control_did', subfolder='applications') # Figure 2: Post-only model size distributions post_size = _cache.get('post_size') if post_size: fig, ax = setup_figure() bins = np.linspace(-2, 7, 40) ax.hist(post_size['ctrl_params'], bins=bins, alpha=0.5, label='Control (US/EU)', color='#3498db', density=True) ax.hist(post_size['cn_params'], bins=bins, alpha=0.5, label='Constrained (CN)', color='#e74c3c', density=True) ax.axvline(np.log(3), color='green', linestyle='--', alpha=0.7, label='3B threshold') ax.axvline(np.log(7), color='orange', linestyle='--', alpha=0.7, label='7B threshold') ax.set_xlabel('log(Parameter Count, Billions)') ax.set_ylabel('Density') beta = post_size['beta'] p = post_size['p'] ax.set_title(f'Post-Treatment Model Size: CN 1.55x Larger ' f'($\\beta$={beta:.2f}, p={p:.4f})') ax.legend(fontsize=8) paths += save_figure(fig, 'export_control_post_size', subfolder='applications') # Figure 3: Edge share comparison at 3B threshold edge_3b = _cache.get('edge_3b') if edge_3b: fig, ax = setup_figure(width=7, height=5) groups = ['Constrained\n(CN)', 'Control\n(US/EU)'] shares = [edge_3b['cn_share'], edge_3b['ctrl_share']] paper_claims = [0.47, 0.25] x = np.arange(len(groups)) width = 0.3 bars1 = ax.bar(x - width/2, shares, width, label='Data', color=['#e74c3c', '#3498db'], alpha=0.8) bars2 = ax.bar(x + width/2, paper_claims, width, label='Paper claim', color=['#e74c3c', '#3498db'], alpha=0.3, edgecolor='black', linestyle='--') ax.set_ylabel('Edge-Compatible Share (<=3B)') ax.set_title(f'Paper Claim vs Data: REVERSED\n' f'$\\chi^2$={edge_3b["chi2"]:.1f}, p={edge_3b["p"]:.4f}') ax.set_xticks(x) ax.set_xticklabels(groups) ax.legend() # Annotate bars for bar, val in zip(bars1, shares): ax.text(bar.get_x() + bar.get_width()/2, bar.get_height() + 0.01, f'{val:.1%}', ha='center', va='bottom', fontsize=10, fontweight='bold') for bar, val in zip(bars2, paper_claims): ax.text(bar.get_x() + bar.get_width()/2, bar.get_height() + 0.01, f'{val:.0%}', ha='center', va='bottom', fontsize=9, color='gray') ax.set_ylim(0, 0.6) paths += save_figure(fig, 'export_control_edge_3b', subfolder='applications') return paths if __name__ == '__main__': results = main() for r in results: print(f"{r['name']}: {r['verdict']} " f"(stat={r['statistic']}, p={r['p_value']})")

r2	0.0226
n obs	171
p value	0.9771
statistic	-0.0061

Export Control DID Edge Share

Description

Figures

Statistics

Script

▸ Source Code

Reproduction Steps