
    c@jZ                         d Z ddlZddlZddlZddlZddlmZ ddlmZ ddlm	Z	 ej                  j                  d e ed      j                  e      j!                         j"                               ddlZ ed      Zd ej*                  _         ej.                  ej0                  d	
        ej2                  d      Zd Zd Zd Zd Zd Zd Z d Z!d Z"d Z#d Z$d Z%d Z&d Z'd Z(d Z)d Z*e+dk(  r e*        yy)u  
build_data.py — Gera os JSONs do dashboard Vendas Setorial a partir do BigQuery.

Pipeline:
  1. Le crosswalks (segmentos.json, cnae_to_segmento.csv, ncm2_to_segmento.csv).
  2. Roda as queries de sql/ no BQ (cliente real, sem mock).
  3. Aplica crosswalks (CNAE->segmento com override por classe; NCM2->segmento).
  4. Trimestraliza RF e MDIC com shares do BCB e projeta ate 2026T1.
  5. Escreve data/*.json e valida somatorios.

Uso:  python3 build/build_data.py
Saida: data/segmentos.json, data/pagamentos.json, data/faturamento.json,
       data/ecommerce.json, data/combinada.json, data/meta.json

Idempotente: re-rodar regenera os JSONs do zero. Nao escreve nada no BQ.
    N)datetime)ZoneInfo)bigquerypathlibzAmerica/Sao_Pauloc                  P    t        j                  t              j                         S N)r   nowTZ	timetuple)as    F/mnt/c/claude-workspace/dashboards/vendas_setorial/build/build_data.py<lambda>r       s    b)9)C)C)E     z)%(asctime)s - %(levelname)s - %(message)s)levelformat
build_datac                  T    t        j                  t              j                  d      S )Nseconds)timespec)r   r	   r
   	isoformat r   r   now_brr   %   s    <<%%y%99r   c                     t        t        j                  dz  d      5 } t        j                  |       d   }d d d        D cg c]  }|d   	 }}t        |      D ci c]  \  }}||
 }}}|||fS # 1 sw Y   >xY wc c}w c c}}w )Nsegmentos.jsonutf-8encoding	segmentoscodigo)openC	XWALK_DIRjsonload	enumerate)fsegsscodesicidxs          r   load_segmentosr-   *   s    	akk,,w	? )1yy|K()"&'QQx['E'%e,
-DAq1a4
-C
-	) )'
-s   A5B B5A>c                     i i }} t        t        j                  dz  d      5 }t        j                  |      D ]  }|r*|d   j                  d      s|d   j                         dk(  r0|d   j                         |d   j                         |d   j                         |d	   j                         f\  }}}}|d
k(  r||f| |<   |dk(  s||f||<    	 ddd       | |fS # 1 sw Y   | |fS xY w)zURetorna (base_por_divisao{div2:(seg,tipo)}, override_por_classe{classe4:(seg,tipo)}).zcnae_to_segmento.csvr   r   r   #r            divisaoclasseN)r    r!   r"   csvreader
startswithstrip)baseoverrider&   rowr   codesegtipos           r   load_cnae_xwalkr?   2   s    (D	akk22W	E -::a= 	-C#a&++C0CFLLNg4M%(V\\^SV\\^SV\\^UXYZU[UaUaUc%c"E4d	!!4[T
(""%t	-- >- >s   B'C!	C!!C-c                     i } t        t        j                  dz  d      5 }t        j                  |      D ]  }|r*|d   j                  d      s|d   j                         dk(  r0t        |      dkD  r|d   j                         nd}|d	   j                         |f| |d   j                         j                  d      <    	 d
d
d
       | S # 1 sw Y   | S xY w)z%Retorna {ncm2: (seg, nome_capitulo)}.zncm2_to_segmento.csvr   r   r   r/   ncm2r1    r0   N)	r    r!   r"   r5   r6   r7   r8   lenzfill)mr&   r;   nomes       r   load_ncm_xwalkrG   A   s    
A	akk22W	E @::a= 	@C#a&++C0CFLLNf4L%(X\3q6<<>rD*-a&,,.$)?Ac!flln""1%&		@@ H@ Hs   B#CCc                 8    | sydj                  d | D              S )z@'47.11-3' -> '4711' ; '011' -> '011' ; '47' -> '47'. So digitos.rB   c              3   B   K   | ]  }|j                         s|  y wr   )isdigit).0chs     r   	<genexpr>zcnae_digits.<locals>.<genexpr>Q   s     5"

25s   )join)r<   s    r   cnae_digitsrO   M   s    775555r   c                     t        |      }| dk(  r#|dd }||v r||   S |j                  |dd d      S |j                  |dd d      S )zNResolve um codigo CNAE para (segmento, tipo): classe override -> divisao base.r4   N   r1   )423servicos)rO   get)granularityr<   r9   r:   dc4s         r   map_cnaerX   T   s`    DAhrU>B<xx"123388AbqE.//r   c                 (   t         j                  |z  j                  d      }t        j	                  d|       t        | j                  |t         j                        j                               }t        j	                  d|t        |             |S )Nr   r   zRodando %s ...)locationz  %s -> %d linhas)
r!   SQL_DIR	read_textloginfolistqueryBQ_LOCATIONresultrC   )clientnamesqlrowss       r   run_sqlrg   a   sk    99t
&
&
&
8CHHt$S1==9@@BCDHH $D	2Kr   c                     d| d    d| dd  S )z'20241' -> 'T1/24'.TrQ   /r1   r   )tris    r   q_labelrl   i   s    s1vhaAaz""r   c           	      :   t        | D ch c]  }|d   	 c}      }t        |      D ci c]  \  }}||
 }}}dD ci c]  }|g  }}d}	| D ]o  }|d   |vr|	dz  }	|d   j                  ||d             |d   j                  ||d             |d   j                  t        j                  j                  |d	   d
             |d   j                  t        j                  |d             |d   j                  t        j                  |d             |d   j                  t        j                  |d             |d   j                  t        t        |d         d
             |d   j                  t        t        |d         d
             |d   j                  t        t        |d         d             r |	rt        j                  d|	       t        |d         }
||D cg c]  }t        |       c}||
d|
fS c c}w c c}}w c c}w c c}w )z#Cubo compacto em colunas paralelas.	trimestre)	r(   qr&   pprtvntwr   segmentor0   r(   ro   r&   funcaor1   rp   payrq   presrr   tierrs   valorrt   qtdru   tarifa_x_valorrQ   zApagamentos: %d linhas com segmento fora do dicionario (ignoradas))quartersquarter_labelscube_total_valor)sortedr%   appendr!   
FUNCAO_IDXrT   PAY_IDXPRES_IDXTIER_IDXroundfloatr]   warningsumrl   )rf   seg_idxrr~   r*   rr   q_idxkcolsskippedtotal_vs              r   build_pagamentosr   o   s   t4!q~45H'12daQT2E2JKaArEKDKG @Z='qLGS	:/0S	q~./S	))!H+q9:S	1U8,-T
!**QvY/0S	AfI./S	uQwZ0!45S	uQuX23T
%a(8&9 :A>?@ WY`a$s)nG/78!71:8	
  ) 52K( 9s   HH
H-Hc                    dddd}i }| D ]$  }|d   |d   }}||vs||   |||      kD  s |||<   & t        |      }	t        j                  d|	D ci c]  }|||   
 c}       g d}
d	d
dddd}g d|D ci c]7  }|D ci c]&  }||
D ci c]  }|i  c}|
D ci c]  }|i  c}d( c}}9 c}}}t        |	      |D ci c]  }|i  }}| D ]  }|d   |d   }}|||   k7  rt	        ||d   ||      \  }}|d   }|
D ]G  }t        |||      xs d      }|   |   |   |   j                  |d      |z   |   |   |   |   |<   I |k(  s|d   |f}||   j                  ||d   |d   |dd      }|dxx   t        |d	   xs d      z  cc<    i }|D ]n  }i ||<   D ]b  }dD ci c]J  }||
D ci c]9  }||	D cg c]*  }t        |   |   |   |   j                  |d      d      , c}; c}}L c}}}||   |<   d p |D ci c]]  }|t        ||   j                         D cg c]1  }t        |d   d      dk7  r|d   |d   |d   t        |d   d      d3 c}d       _ }}}t        fd|D              }|	|
|||	D ci c]  }|||   
 c}|dS c c}w c c}w c c}w c c}}w c c}}}w c c}w c c}w c c}}w c c}}}w c c}w c c}}w c c}w )z`Escolhe granularidade mais fina por ano; mapeia p/ segmento; series por regime e drill por CNAE.r   r0   r1   )r3   grupor4   anorU   zRF granularidade por ano: %s)receitan_cnpjmassaexportarrecreceita_brutar   massa_salarialexportacoesarrecadacao_total)varejoatacado	industriarS   normalsimples	cnae_coderegime        	cnae_desc)r<   descr>   r   r   r<   r   r>   c                     | d    S )Nr   r   xs    r   r   z#build_faturamento.<locals>.<lambda>   s    1Y<- r   keyc              3   v   K   | ]0  }D ])  }d D ]"  }|   |   |   d   j                  d       $ + 2 yw)r   r   r   NrT   )rK   r(   tpregseriestiposults       r   rM   z$build_faturamento.<locals>.<genexpr>   s]      U!6K/2 	q	"c9%))#s333s   69)anosmetricasr   r   drill_finest_ult_ano_total_receita_ult)r   r]   r^   maxrX   r   rT   
setdefaultr   valuesr   )rf   	seg_codesr9   r:   orderfinestr   r   gr   metrics
metric_colr(   r   rE   r   r=   r   rs   r   rV   
out_series	out_drilltotal_rbr   r   r   s                           @@@r   build_faturamentor      s    A3EF x=)1F?eAhvay)99F1I &>DHH+D-Iqal-IJ?G"hAQ*=J ;E   	
dij^`B73aArE3PW@X1B@XYYjjF
 d)C%&qQU&E& ;x=)1q	>1andH=Rk 	QAa
1&+!,A)/R)=a)@)D)DQ)Lq)PF3KOC #A&	Q 8[>2&Cc
%%cAkNAkNdfsv+wxAiLE!O"4"9::L; J 
1 	B 1! ! ^efYZatT!%q	"c 21 5 9 9!S A1ETTff!JqM"	   	
6Ahoo'J59q+AS+H i6AfIRWXYZcXdfgRhi J'
 	
I    H
 *./QAvayL/&	 	g .J 4@Xj '* Uf!J$ 0s   J5
>K
	
K
J:K#
J?-K4K

K:K!
K/K>KK! K-=6K(3K-!K3:
KK
KK!(K-c                    t        | D ch c]  }|d   	 c}      }t        |      |D ci c]  }|i i d
 c}|D ci c]  }|i  }}i }| D ]  }t        |d         j                  d      }|j	                  |d      \  }	}
|
||<   |d   }|d   }t        |d   xs d      }|	   |   j	                  |d	      |z   |	   |   |<   |k(  s||	   j	                  |d	      |z   ||	   |<    |D ci c]D  }|dD ci c]3  }||D cg c]$  }t        |   |   j	                  |d	      d      & c}5 c}}F }}}}|D ci c]S  }|t        ||   j                         D cg c]%  \  }}||j	                  |d
      t        |d      d' c}}d       U }}}}t        fd|D              }||||dS c c}w c c}w c c}w c c}w c c}}w c c}}}w c c}}w c c}}}w )Nr   r   rA   r1   )rR   rB   r   valor_brutor   r   rB   )rA   rF   r{   c                     | d    S )Nr{   r   r   s    r   r   z!build_ecommerce.<locals>.<lambda>   s    1W:+ r   r   c              3   \   K   | ]#  }d D ]  }|   |   j                  d        % yw)r   r   Nr   )rK   r(   r   r   r   s      r   rM   z"build_ecommerce.<locals>.<genexpr>   s4     aK`aCq	#""3,a,as   ),)r   r   r   r   
_total_ult)	r   r   strrD   rT   r   r   itemsr   )rf   r   ncm_mapr   r   r(   r   ncm_nomerA   r=   rF   r   r   rs   r   r   r   totalr   r   s                     @@r   build_ecommercer      s=   T*1U8*+D
d)C8AB1aB2..BF%&qQU&E&H 	=1V9~##A&KKk2	TkeH!M"'a($Sk#.221c:Q>sC8$Sz~~dC81<E#Jt	=   	
Qfg#CE1%q	#**1c2A6EEggJ   
 	 	
6X]^_X`XfXfXhiPTPQSTaa!4uQ{Ki%
 	
I  aiaaE 7 +B& Fg jsR   GG	
G8G
G)G6G;G!G+,*G%G+GG%G+c                 	  %& |D ci c]  }|i  c}%t               }| D ]M  }|d   }|%vrt        |d   xs d      t        |d   xs d      f%|   |d   <   |j                  |d          O d &g }t        t        j
                  d      D ]  }| &|      z  } |t        j                  z  }|j                  t        j                  d         }	t        t        t        j
                  t        j                  dz               }
g d	}fd
}fd}d%&fd	}d%fd	}&fd}i }|D ]u  }i }|D ],  }%|   j                  |      }|rt        |d   d      nd||<   . |D ci c]  }|i  }}|D ]  }||   }t        t        j
                  t        j                  dz         D ]I  } ||||      } |||d      }t         &|            D ]  \  }}|t        |||   z  d      nd||<     K t        j                  D ]L  }t        |dd       |d   }}|j                  |dz
   |       }|t        | |||d      z  d      nd||<   N  i }t        t        j
                  t        j                  dz         D ]H  } |||      } |||d      }t         &|            D ]  \  }}|t        |||   z  d      nd||<     J t        j                  D ]L  }t        |dd       |d   }}|j                  |dz
   |       }|t        | |||d      z  d      nd||<   N ||D cg c]  }t        |       c}|	|D cg c]  }|j                  |       c}|D cg c]  }|j                  |       c}|D ci c]%  }||D cg c]  }||   j                  |       c}' c}}d} |
D cg c]  } |||       }!}g }"|D ci c]  }|g  }#}|
D ]  }|t        j                  k  r<|"j                   |||             |D ]  }|#|   j                   ||||               R|"j                   |||             |D ]   }|#|   j                   |||   |             "  |
|
j                  t        j                  dz         |!|"|#d}$| |$d||<   x |t        j                  |dS c c}w c c}w c c}w c c}w c c}w c c}w c c}}w c c}w c c}w )z?Trimestraliza RF e MDIC com shares do BCB e projeta ate 2026T1.rv   valor_totalr   valor_nao_presentern   c                 4    dD cg c]  }|  | 
 c}S c c}w )N)r0   r1   r2   rQ   r   )yro   s     r   quarters_of_yearz)build_combinada.<locals>.quarters_of_year  s    #/0a1#aS	000s   i  r1   )todosr   r   r   rS   c                     |d   vryd   j                  |      }|dk(  rd   n|g}d}|D ].  }|d   |    |   d   d   |   d   |    |   d	   d   |   z   z  }0 |S )
zXReceita RF consolidada (normal+simples) por (seg, ano, tipo). tipo='todos' soma os elos.r   Nr   r   r   r   r   r   r   index)r=   r   r>   r*   elostotr   fats          r   	rf_annualz"build_combinada.<locals>.rf_annual  s    c&k!Kc"#ws7|TF 	tB3x=%b)(3I>qACMRUDVWYDZ[dDefoDpqrDsssC	t
r   c                 v    |d   vry d   j                  |      }d   |    d   |   d   |    d   |   z   S Nr   r   r   r   r   r=   r   r*   ecoms      r   mdic_annualz$build_combinada.<locals>.mdic_annual!  sX    d6l"Ls#H~c"8,Q/$x.2Ei2PQR2SSSr   Fc                     g } 	|      D ]2  }|    j                  |d      }|j                  |r|d   n|d          4 t        |      }|dk  rg dS |D cg c]  }||z  	 c}S c c}w )zRShare de cada trimestre do ano (lista de 4). Fallback igual (0.25) se ano sem BCB.r   r   r0   r   )      ?r   r   r   )rT   r   r   )
r=   r   npvalsrr   tvr   rs   bcbr   s
           r   	bcb_sharez"build_combinada.<locals>.bcb_share'  sx    !#& 	0ASa,BKK1A/	0 $i!8++!%&AC&&&s   A(c                     t        |dd       |d   }}	|    j                  |d      }	|    j                  |dz
   | d      }|r|d   n|d   }|r|d   n|d   }|dk  ry||z  S )z;Razao YoY do BCB: valor[tri]/valor[tri-4ano]. Fallback 1.0.NrQ   r   r0   r         ?)intrT   )
r=   rk   r   r   ro   curprevcvpvr   s
            r   	bcb_ratioz"build_combinada.<locals>.bcb_ratio2  s    3r7|SV1#hll3
+3x||qseA3K4SVs1vT!WQ7Bwr   c                      |      D cg c]  }| j                  |       }}t        d |D              ry t        t        d |D              d      S c c}w )Nc              3   $   K   | ]  }|d u  
 y wr   r   rK   rs   s     r   rM   z9build_combinada.<locals>.annual_from_q.<locals>.<genexpr>?  s     %QqDy%s   c              3   &   K   | ]	  }||  y wr   r   r   s     r   rM   z9build_combinada.<locals>.annual_from_q.<locals>.<genexpr>A  s     8q!-8s   r1   )rT   allr   r   )qmapr   rr   qsr   s       r   annual_from_qz&build_combinada.<locals>.annual_from_q=  sQ    #3C#89adhhqk99%"%%S8B88!<< :s   ANr0   )r   rQ   T)r~   r   proj_from_idxr   mdicrf)r   r   r   r  r  )
trimestralanual)r   _proj_quarterstipos_rf)F)setr   addranger!   ANO_INICIO_COMBINADAQUARTERS_PROJr   r_   ULTIMO_ANO_ANUALrT   r   r%   r   rl   r   )'shares_rowsr   r   r   r(   all_trisr   tri_axisr   r   ano_axisr  r   r   r   r   r   r   bcb_qrr   r   r   rf_q_by_tiporfqrf_ashr   ro   r   mdic_qmdic_ashn
tri_seriesbcb_anmdic_anrf_an	an_seriesr   r   s'    ``                                  @@r   build_combinadar     sb    $
$Q1b5
$CuH %jMC<"'-(8(=A">aH\F]Fbab@c!dAq~Q{^$%1 H1))40 ($Q''(HNN1??1#56M E!00!2D2Dq2HIJHFH	T	'	= F ;C 	7AQAB*,uRUA$E!H	7
 *222B22 
	bBr"C11113E3E3IJ R Ar*q!.%&6q&9: RDAq7;7GU4"Q%<3TCFRR
 __ b1Ra5z1Q41ww!A#qc{+GKGWti1&??C]aAb
	b q--q/A/AA/EF 	VA A&FAqT*C!"21"56 V19?9KE&3q6/15QUq	V	V
  	`Aq!u:qtqA::1aSk*DEIEUdYq!%==qA[_F1I	` !3;<awqz<**23QEIIaL3,45qVZZ]5MUVr2BAR(,,Q/BBV

 4<<a-q)<<"*+BR++ 	IAA&&&{1a01" :B"I$$Yq!R%89: }VQ78" IB"I$$]<3CQ%GHI	I %^^A,>,>,BC7%
	
 $.	Bq	w;Cz XVVG %\ 36 =35BV =+sA   
R;)
S 3SS
*S	
SS.S=S
S$Sc                    j                  di       }d   D cg c]  }|j                  |      dk(  s| }}|sd   }d   }ddddd}g d	fd
}	|D 
ci c]  }
|
 |	|       c}
|D 
ci c]  }
|
 |	|       c}
d| D ]  }|d   }|d   vr|d   |d   |d   |d   |d   f\  }}}}}t        |d   xs d      }g }||v r|j                  d       |dk(  r||v r|j                  d       |D ]  }|   |   }|d   |xx   |z  cc<   |dk(  ro|d   |xx   |z  cc<   |dv r|d   |xx   |z  cc<   |dk(  r|d   |xx   |z  cc<   n|d   |xx   |z  cc<   |j                  |      }|s~||   |xx   |z  cc<   |dv s|d    |xx   |z  cc<     fd!}fd"}fd#}||i i d$}|D ]$  }
 |d|
||      |d   |
<    |d|
||      |d   |
<   & |S c c}w c c}
w c c}
w )%u(  Penetracao de cartao por segmento (anual, sem projecao).
    'total': todas as capturas / faturamento das empresas (RF).
    'ecommerce': cartao nao presente / faturamento e-commerce (MDIC).
    Componentes do numerador permitem os drills credito/debito, tier, a vista x parcelado e nº parcelas.r   r   r4   p1p23p46p7)	1 parcelaz2 e 3 parcelasz4 a 6 parcelasz7 ou + parcelas)
cardcreditodebito_outroscred_prem_empcred_avistacred_parcelador   r!  r"  r#  c           
      ^    D ci c]  }|| D ci c]  }|d c} c}}S c c}w c c}}w )Nr   r   )r   r   r   comps      r   blankzbuild_mercado.<locals>.blank  s.    378aD)qAsF))88)8s   
)
$)))r   	ecommercerv   r   r   rw   ry   rz   parcelasr{   r   nao_presenter.  r%  u   Créditor&  )premiumcorpr(  r$  r)  r*  )u   Débitou	   Pré-pagor'  c                 \     d   j                  |      t         fddD              S )zWDenominador da aba mercado = faturamento de VAREJO apenas (consolidado normal+simples).r   c              3   F   K   | ]  }d       d   |   d        yw)r   r   r   Nr   )rK   rgr   r*   r=   s     r   rM   z3build_mercado.<locals>.rf_varejo.<locals>.<genexpr>  s.     bb3x=%h/3I>qAbs   !r   )r   r   )r=   r   r*   r   s   ` @r   	rf_varejoz build_mercado.<locals>.rf_varejo  s)    Kc"bLabbbr   c                 f    d   j                  |      }d   |    d   |   d   |    d   |   z   S r   r   r   s      r   
mdic_totalz!build_mercado.<locals>.mdic_total  sI    Ls#H~c"8,Q/$x.2Ei2PQR2SSSr   c           
          |    |   }d|D cg c]  }t         |||      d       c}i}	D ]%  }|D cg c]  }t        ||   |   d       c}||<   ' |S c c}w c c}w )Nr   r1   )r   )
scoper=   r   fat_fnr   anblockr   accr,  s
           r   emitzbuild_mercado.<locals>.emit  sr    JsOTBrvc22BC 	;A59:rad2h*:E!H	; C:s
   AA)
anos_total	anos_ecomr   r.  )rT   r   r   )rows5r   r   r   r   r   r@  rA  PARCr-  r(   r   r=   r   rw   ry   rz   parcrs   scopesscpkr6  r8  r?  outr>  r,  s    ``                       @@r   build_mercadorI    s    WWY#F [FFJJqMX,E!FJF[
VI%dhiD&D9 4==aQj))=6?@E),,@BC  -
mc'l"()%!H+qy!F)UVWaUb(b%VT4!G*/"*MM'">!cY&6MM+& 	-BBAfIcNaN#)S!Q&!..o&s+q0+;&m$S)Q.)&',1,XXd^bE#J!OJ33/"3'1,'	--8c
T $)bWY
ZC Jw:yAGQ";9jIKJ J} G >@s   G(G(G-4G2c           	      6   t         j                  d       d}|d   }|d   |   d   }|d   j                  d      }	|d   d	   |	   }
|d   |   d
   }t        d t	        |d   |d   d	         D              }|
r/t        ||
z
        |
z  dkD  rt         j                  d||
|       d}nt         j                  d|       |d   |   d
   d   d   dk7  rt         j                  d       d}nt         j                  d       fd d      }t        fd|D              }t        d         dz
  }d   D ci c]  }|d c}|D ]@  }d   D ]6  }|xx   d   |   |   d   d   |   d   |   |   d   d   |   z   z  cc<   8 B t        j                               xs dt         j                  ddj                  fd d   D              z          |r||z  nd!}t         j                  d"|d#z         |d$kD  rt         j                  d%       t         j                  d&t        | d'   d(         t        | d                t         j                  d)t        |d         |d*   d+z         d,}|d-   |   }t        |d.         dz
  }t        |d/   |   |d0   |   z   |d1   |   z
        }t        |d2   |   |d3   |   z   |d/   |   z
        }t        |d4   |   |d5   |   z   |d6   |   z   |d7   |   z   |d/   |   z
        }t        |||      d8kD  rt         j                  d9|||       d}nt         j                  d:       |d;   |   r|d1   |   |d;   |   z  nd!}t         j                  d<|d.   |   |d#z  t        |d=                |S c c}w )>Nz=== Validacao ===T   r   r  r   i  r  r   r  c              3   P   K   | ]  \  }}|j                  d       s||   yw)2024N)r7   )rK   rr   rs   s      r   rM   zvalidate.<locals>.<genexpr>  s)     wDAqPQP\P\]cPdijivaws   &&&r~   gMbP?z8  FAIL: soma trimestral RF 2024 (%s) != anual (%s) p/ %sFz*  OK: trimestral RF 2024 == anual (seg %s)20261z-  FAIL: serie trimestral nao termina em 20261z  OK: projecao chega a 2026T1c                 6     t         fdd   D              S )Nc              3   `   K   | ]%  }d D ]  }t        d      |   |   d            ' yw)r   r   r   Nr   )rK   r   r5  r   sgs      r   rM   z+validate.<locals>.rb_seg.<locals>.<genexpr>  sI      
-B
') Hb!"%b))45
5
s   +.r   rR  )rS  r   s   `r   rb_segzvalidate.<locals>.rb_seg  s!     
'l
 
 	
r   rR   c              3   .   K   | ]  } |        y wr   r   )rK   rS  rT  s     r   rM   zvalidate.<locals>.<genexpr>  s     00s   r0   r   r   r   r   r   z  Cadeia (ult ano): u    · c              3   B   K   | ]  }| d |   z  dz  dd  yw)=d   z.0f%Nr   )rK   r   by_tipotot_tipos     r   rM   zvalidate.<locals>.<genexpr>  s2     1q^`RD'"+h:Ns:RSV9WWX2Y1qs   r   z@  Cobertura: segmento 'Outros'(423) = %.1f%% da receita total RFrX  gffffff?z<  ATENCAO: 423 concentra >35%% da receita; revisar crosswalkz.  pagamentos: %d combos no cubo, %d trimestresr   rs   z.  ecommerce: %d anos, total ult ano R$ %.1f bir   g    eA416r   r@  r&  r'  r%  r)  r*  r   r!  r"  r#  r   zE  FAIL: identidades de cartao nao fecham (id1=%.2f id2=%.2f id3=%.2f)za  OK: identidades cartao fecham (credito+debito=card; avista+parcelado=credito; parcelas=credito)r   zG  Mercado: penetracao cartao seg 416 (%d) = %.1f%% | e-commerce %d anosrA  )r]   r^   r   r   zipabserrorrC   r   rN   r   r   )pagr   r   combmercr   oksampler(   i2024rf_anual_2024rk   rf_tri_2024rb423rb_allult_ir   rS  share423smpbjid1id2id3penrZ  rT  r[  s    `                        @@@r   validaters    s   HH !	Br]FXvw'AfIOOD!EdGG$U+M
x.
 
.CwCJT79K$LwwK[=89MIEQ		Lk[hjpq=vFH~fl+J7;wF		AB01

 5ME0i00FFq E!$W.2r3w.G Bg, 	BBBK3x=,R0:9EeLsS[}]_O`acOdenOopyOz  |A  PB  B  BK	BB 7>>#$)HHH#fkk1qdghodp1q&qqr!'uv~QHHHOQY\_Q_`$RSHH=s3v;sCS?TVYZ]^hZiVjkHH=s4<?PRVWcRdgjRjk
CWcAD!#A
q|A?!3A!66!F)A,F
GC
q"Q'7%8%;;q|AN
OC
qwqzAeHQK'!E(1+5$
Ba	lSToU
VC
3SC		Y[^`cehitu()%!F)A,5!
$CHHV,"C#Is43D/EGI7 /s   <
Nc                 z    | j                         D ci c]  \  }}|j                  d      r|| c}}S c c}}w )N_)r   r7   )rV   r   rs   s      r   strip_privaterv    s/    WWY@TQall3.?AqD@@@s   77c            	         t         j                  dt                      t               \  } }}t	               \  }}t               }t         j                  dt        |      t        |      t        |      t        |             t        j                  t        j                  t        j                        }t        |d      }t        |d      }t        |d      }	t        |d      }
t        |d      }t        ||      \  }}t        ||||      }t        |	||      }t!        |
|||      }t#        ||||      }t%        ||||||      }t        j&                  j)                  d	
       t               t        j*                  t        j,                  t        j.                  t        j0                  dt        j2                  D cg c]  }t5        |       c}d}d| it7        |      t7        |      t7        |      |||d}|j9                         D ]s  \  }}t        j&                  |z  }|j;                  t=        j>                  |dd      d       t         j                  d||jA                         jB                  dz         u t         j                  dt               |rdnd       tE        jF                  |rd       y d       y c c}w )NzBuild iniciado %szQCrosswalks: %d segmentos, %d divisoes base, %d overrides classe, %d capitulos NCM)projectrZ   zv1_pagamentos_cube.sqlzv2_faturamento.sqlzv3_ecommerce.sqlzv4_bcb_shares.sqlzv5_cartao_anual.sqlT)exist_ok)
pagamentosfaturamentor.  fonte)	gerado_emrotulosproj_quartersr   )r   zpagamentos.jsonzfaturamento.jsonzecommerce.jsonzcombinada.jsonzmercado.jsonz	meta.jsonF),:)ensure_ascii
separatorsr   r   zEscrito %s (%.1f KB)i   z!Build concluido %s | validacao=%sOKFALHOUr   r1   )$r]   r^   r   r-   r?   rG   rC   r   Clientr!   
BQ_PROJECTra   rg   r   r   r   r  rI  rs  DATA_DIRmkdirROTULO_PAGAMENTOSROTULO_FATURAMENTOROTULO_ECOMMERCEFONTE_PUBLICAr  rl   rv  r   
write_textr#   dumpsstatst_sizesysexit)r'   r   r   r9   r:   r   rc   pag_rowsfat_rows	ecom_rowsr  cartao_rowsr`  ru  r   r   ra  rb  rc  rr   metarH  fnamepayloadrp   s                            r   mainr    sG   HH &(+-/D)W$&ND(GHH`^SYHs7|E __Q\\AMMJFv78Hv34H 23I&"56K&"78Kh0FC
Hix
@C9i9D;T9=Dc4;D	#sD$i	8BJJd#X"#"5"5aFZFZ!"!3!3aooO./oo>'!*>	D '-(-)#.'-C ))+ IwJJ	TZZe
S^ef'0@0@40GHI
 HH0&(BDHUHH"Q!# ?s   ?K__main__),__doc__r5   r#   loggingr  r   zoneinfor   google.cloudr   pathinsertr   
__import__Path__file__resolveparentconfigr!   r
   	Formatter	converterbasicConfigINFO	getLoggerr]   r   r-   r?   rG   rO   rX   rg   rl   r   r   r   r  rI  rs  rv  r  __name__r   r   r   <module>r     s       
   ! 3z),11(;CCELLM N  !"E      ',,/Z [g%:
	6	0#<FR"JFWRGV:zA.b zF r   