
    ^jB5                        d dl mZ d dlZd dlZd dlZd dlZd dlmZ d dlm	Z	 d dl
mZmZmZ d dlmZ d dlmZmZ d dlmZ dd	Zdd
ZddZddZddZdddZddZdddd dZd!dZd"dZd#dZddd$dZ y)%    )annotationsN)Path)Any)assess_completenessextract_legal_fragmentidentify_product)connect)init_core_schemautc_now)get_snapshotc                0    t        j                  | d      S )NF)ensure_ascii)jsondumps)values    =/var/www/origus_pro_usr/data/www/origus.pro/core/knowledge.py_jsonr      s    ::e%00    c                   | j                  d      }|st        d      t        |      }|j                         st        d      |j	                  dd      j                         }t        |      dk  rt        d      |S )	N	text_pathu6   У версии нет TXT-представления.u,   TXT-файл версии не найден.utf-8replace)encodingerrors   uG   TXT версии слишком короткий для анализа.)getRuntimeErrorr   exists	read_textstriplen)snapshotrawpathtexts       r   _read_snapshot_textr&      sq    [!Cl#[\\	cD;;=-[ \\			:	@	@	BD
4y}L)rssKr   c                    g }t        j                  dt         j                        }| j                         D ];  }|j	                         }|s|j                  |      s(|j                  |d d        = |S )Nuq   ^\s*(статья\s+\d+[\w.-]*|глава\s+\w+|раздел\s+[IVXLC\d]+|приложение\s*№?\s*\d+)\b.*$i,  )recompileI
splitlinesr    matchappend)r%   resultpatternlines       r   _heading_linesr1      sz    FJJ  L  MO  MQ  MQ  RG! CZZ\GMM$'tDSz)BC Mr   c                (  
 t        |       }t        j                  d|       D ch c]  }t        |       c}t        j                  d|       D ch c]  }t        |       }}t        j                  d|       D ch c]  }t        |       c}
t        j                  d|       D ch c]  }t        |       }}t	        fd|D              }t	        
fd|D              }g }|r4|j                  ddj                  t        t        |d d	             z          |r4|j                  d
dj                  t        t        |d d	             z          |r|d   nd}t        |xs |      }	|	rdnd|	rdnd|||||dS c c}w c c}w c c}w c c}w )Nu   (?im)^\s*статья\s+(\d+)\bu'   (?i)стать(?:е|и|ю|я)\s+(\d+)\bu.   (?im)^\s*приложение\s*№?\s*(\d+)\bu6   (?i)приложени(?:е|я|ю|и)\s*№?\s*(\d+)\bc              3  ,   K   | ]  }|vs|  y wN ).0xarticle_headingss     r   	<genexpr>z&_local_completeness.<locals>.<genexpr>.   s     Q!q@P7PAQ   	c              3  ,   K   | ]  }|vs|  y wr4   r5   )r6   r7   appendix_headingss     r   r9   z&_local_completeness.<locals>.<genexpr>/   s     UA!CT:TaUr:   uT   В тексте есть ссылки на отсутствующие статьи: z,    u\   В тексте есть ссылки на отсутствующие приложения:  
incompleteunknowng333333?g      ?)statusscorereasonslast_visible_sectionheadingsmissing_articlesmissing_appendices)
r1   r(   findallintsortedr-   joinmapstrbool)r%   rF   r7   article_refsappendix_refsrG   rH   rD   lastdefiniter<   r8   s             @@r   _local_completenessrT   '   s   D!H&(jj1STX&YZc!fZ"$**-WX\"]^Q#a&^L^')zz2cdh'ij!s1vj#%::.ghl#mna3q6nMnQ|QQUUUGmnrnwnwx{|  AQ  RU  SU  AV  yW  oX  X  	Yuvzvv  AD  EH  I[  \_  ]_  I`  Aa  wb  b  	c!"rD"8&89H!)y c#+/  [^jns   F F?F
,Fc                :   dj                  |j                  d      xs g       }d| d d z   dz   |d d z   dz   dz   t        j                  |j                  d	      xs g       z   dz   d
z   t        j                  |j                  d      xs g       z   dz   | dd  z   S )N
rF   u   НАЧАЛО КОПИИ:
i  u)   

СТРУКТУРА/ЗАГОЛОВКИ:
iX  u'   

ЛОКАЛЬНЫЕ ПРОВЕРКИ:
zmissing_articles=rG   zmissing_appendices=rH   u   

КОНЕЦ КОПИИ:
i)rL   r   r   r   )r%   localrF   s      r   _digestrX   C   s    YYuyy,23H$T%4[06	77?	G4	5 		 !JJuyy1C'D'JK	L MQ	Q 			 #jj3G)H)NBO		P
 	'	'
 (,EF|	4r   c                H   t        j                  d|       }g }g }d}|D ]  }|j                         }|st        |      |kD  rY|r$|j	                  dj                  |             g }d}t        dt        |      |      D ]  }|j	                  ||||z            }t        |      dz   }	|r7||	z   |kD  r/|j	                  dj                  |             |g}t        |      }|j	                  |       ||	z  } |r |j	                  dj                  |             |xs | gS )Nz\n\s*\nr   z

   )r(   splitr    r!   r-   rL   range)
r%   	max_chars
paragraphschunkscurrentsize	paragraphpiadds
             r   _chunksrf   O   s   xx
4(JFrw4 )	
//
(q6)fkk'23RWa1SVI.Oa!I+>N0OOF1HtCx	)MM&++g./!3q6dNN1tSyt) fkk'23dVr   c                   t               }g }| D ]  }|j                  dd      t        j                  dd|j                  dd      j	                               j                         t        j                  dd|j                  dd      j	                               j                         f}||v r|j                  |       |j                  |        |S )N	rule_typer?   z\s+ legal_basis	statement)setr   r(   sublowerr    re   r-   )rulesseenoutrulekeys        r   _dedupe_rulesrt   c   s    	DB (XXk"%bffVCr8R8X8X8Z&[&a&a&cVVF3txxB7==?@FFHJ$;szz$'	(
 Jr   F)forcemodelc               	   t                t        |       }|st        d      t               5 }|j	                  d| f      j                         }d d d        r|sddt        |      dS t        |      }t        |j                  d      xs d      }t        |j                  d      xs d      }t        |      }	i }
|	d   d	k(  rd	|	d
   |	d   |	d   d}n2t        ||t        ||	            }|d   }|j                  d      xs i }
g }g }g }g }||ddd}|d   d	k7  r7t        |      }t        |d      D ]  \  }}t        ||||t!        |            }|d   }|dk(  r|j                  d      r|d   }|j#                  |j                  d      xs g        |j%                  |j                  d             |j                  d      xs i }|j%                  |j                  d             |j%                  |j                  d             |j                  d      dk(  s|t!        |      k  st'        j(                  d        t+        |      }t-               }|d   d	k(  rdnd}|r|d   n|
j                  d      }|r|d   n|
j                  d      }t               5 }|j	                  d|d   f       t/        |j                  d       |j                  d!      ||D cg c]  }|s|	 c}|	d"   |	d#   d$d%      }|rtt1        |d&         }|j	                  d'||j                  d      xs ||d(   ||d   t3        |d
         t/        |d         |d   ||||f       |j	                  d)|f       nk|j	                  d*|d   | ||j                  d      xs ||d(   ||d   t3        |d
         t/        |d         |d   |||f      }t1        |j4                        }|D ]V  } |j	                  d+||d   | | d,   | d-   | d.   t/        | d/         t/        | d0         t3        | d1         |r|d   nd |f       X |j	                  d2|d   d	k(  rdnd3t/        ||d   |d
   t!        |      |d4      | f       |j7                          d d d        dd5||t!        |      ||||dk7  rt!        t        |            ndd6d7S # 1 sw Y   }xY wc c}w # 1 sw Y   LxY w)8NuM   Версия нормативного документа не найдена.z<SELECT * FROM knowledge_documents WHERE source_snapshot_id=?T)okcachedknowledge_documentdocument_coder?   titlerB   r@   rC   rD   rE   )rB   rC   rD   rE   )r{   r|   digestdatameta)coder|   document_typerevision   )r{   r|   r%   fragment_indexfragment_totaldocumentro   	ai_run_idproviderrv   groqr=   source_incompletereview_requiredr   zAUPDATE knowledge_documents SET is_current=0 WHERE document_slug=?document_slugr   r   rG   rH   )rG   rH   )r   r   r   
ai_run_idslocal_completenessida.  
                UPDATE knowledge_documents SET document_code=?,title=?,source_hash=?,status=?,
                    completeness_status=?,completeness_score=?,completeness_reasons_json=?,last_visible_section=?,
                    model=?,extracted_at=?,is_current=1,meta_json=? WHERE id=?
            content_hashz9DELETE FROM knowledge_rules WHERE knowledge_document_id=?aR  
                INSERT INTO knowledge_documents(document_slug,source_snapshot_id,document_code,title,source_hash,status,
                    completeness_status,completeness_score,completeness_reasons_json,last_visible_section,model,
                    extracted_at,is_current,meta_json) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,1,?)
            a2  
                INSERT INTO knowledge_rules(knowledge_document_id,document_slug,source_snapshot_id,rule_type,legal_basis,
                    statement,conditions_json,outcomes_json,confidence,review_status,ai_run_id,created_at)
                VALUES(?,?,?,?,?,?,?,?,?,'review_required',?,?)
            rh   rj   rk   
conditionsoutcomes
confidencezQUPDATE source_snapshots SET extraction_status=?,extraction_meta_json=? WHERE id=?ai_review_required)knowledge_document_idcompleteness_statuscompleteness_scorerules_extractedr   F)r   rv   r_   )rx   ry   r   r   completenessr   review_statusr   )r
   r   r   r	   executefetchonedictr&   rN   r   rT   r   rX   rf   	enumerater   r!   extendr-   timesleeprt   r   r   rJ   float	lastrowidcommit)!snapshot_idru   rv   r"   conexistingr%   r   r|   rW   ai_completeness_metar   ai_checkro   r   	providersmodelsdoc_infor_   idxchunkair~   r   nowrB   provider_used
model_usedr7   	meta_jsonknowledge_idcurrr   s!                                    r   analyze_snapshotr   m   sw   +&H<(wxx	 wc[]h\jkttvw4T(^LL	X	&D	X\\/*0b	1D
hll7#)r
*E
d
#E X$!%.I&eDZ>[

 %4e7SWX]K^_f%%\\&17R
EjrY"6%bIH H|+t}"6!, 
	IC%Du%_bruv|r}~BFDAv$((:.j9ILL'*0b1bff[12%2DTXXj12FMM$((7BS4Txx
#V+CK

2
	 E"	C".x"8,"FL]F#,9Q<2F2J2J:2VM#q	)=)A)A')JJ	 (cWYabqYrXtu$LL9X\\R\E]$j2NA12N5:;M5Ndij~d  "A
 	
 Xd^,LKK  hll7+4uXn5MfUabjUk|G,-eL4K.L\ZpMqC	,8	9 KKSUaTcd  /*;tHLL<Q<ZUZ[cdr[stzh'l7.C(DU<XaKbEc45jYP	QC S]]+L 	=DKK  h7DDUVZ[hVijnozj{tL)*5j1A+B5lI[C\",A$s<	=	= 	g,8,BL,P(VjLWcdlWm1=g1FY\]bYco|~  @KL	M 	

Q(T u\U]'#e*U[,Z_egz_zWUY]I[  AB  CD Dqw wd 3O	( (s1   "S
(AS)S1S5FS
SSS%c                    t                t               5 } | j                  d      j                         }d d d        g }D ]t  }t	        |      }t        j                  |j                  d      xs d      |d<   t        j                  |j                  d      xs d      |d<   |j                  |       v |S # 1 sw Y   xY w)Na  
            SELECT kd.*,
              (SELECT COUNT(*) FROM knowledge_rules kr WHERE kr.knowledge_document_id=kd.id) AS rules_count,
              (SELECT COUNT(*) FROM knowledge_rules kr WHERE kr.knowledge_document_id=kd.id AND kr.review_status='review_required') AS rules_review_required
            FROM knowledge_documents kd WHERE kd.is_current=1 ORDER BY kd.document_code
        completeness_reasons_json[]completeness_reasonsr   {}r   )	r
   r	   r   fetchallr   r   loadspopr-   )r   rowsr.   rowitems        r   list_knowledger      s    	 c[[  
 XZ 	 F T#YTZZId@e@mim5n34ZZ 5 =>Vd@ST M s    B::Cc                   t                t               5 }|j                  d| f      j                         }|s
	 d d d        y |j                  d|d   f      j	                         }d d d        t              }t        j                  |j                  d      xs d      |d<   t        j                  |j                  d      xs d      |d	<   g }D ]t  }t        |      }t        j                  |j                  d
      xs d      |d<   t        j                  |j                  d      xs d      |d<   |j                  |       v ||dS # 1 sw Y   xY w)NzaSELECT * FROM knowledge_documents WHERE document_slug=? AND is_current=1 ORDER BY id DESC LIMIT 1zGSELECT * FROM knowledge_rules WHERE knowledge_document_id=? ORDER BY idr   r   r   r   r   r   r   conditions_jsonr   outcomes_jsonr   )r   ro   )
r
   r	   r   r   r   r   r   r   r   r-   )r   r   docro   r   parsedr   rr   s           r   get_knowledger      s   	 }cKK{  ~K  }M  N  W  W  Y4} } kkcehimendpqzz|} #YHHLLQlDmDuqu9v!78ZZ[ 9 ATBHVF p#Y4::dhh?P6Q6YUY+Z\*mqmwmwx|  yA  yA  BQ  yR  yZ  VZ  n[\`ak\l  ]c  ]j  ]j  ko  ]pp//} }s   %E$EEc                    t                t               5 } | j                  d      j                         d   }| j                  d      j                         d   }| j                  d      j                         d   }| j                  d      j                         d   }d d d        t	              t	              t	              t	              dS # 1 sw Y   4xY w)Nz;SELECT COUNT(*) FROM knowledge_documents WHERE is_current=1r   z{SELECT COUNT(*) FROM knowledge_documents WHERE is_current=1 AND completeness_status IN ('possibly_incomplete','incomplete')z{SELECT COUNT(*) FROM knowledge_rules kr JOIN knowledge_documents kd ON kd.id=kr.knowledge_document_id WHERE kd.is_current=1zSELECT COUNT(*) FROM knowledge_rules kr JOIN knowledge_documents kd ON kd.id=kr.knowledge_document_id WHERE kd.is_current=1 AND kr.review_status='review_required')	documentspossibly_incompletero   r   )r
   r	   r   r   rJ   )r   docsr@   ro   pendings        r   summaryr      s	   	 Pc[[VW``bcde;;   ]  ^  g  g  i  jk  l
kk  X  Y  b  b  d  ef  g  A  B  K  K  M  NO  P	P
 D	JPSTYPZmpqxmyzzP Ps   B	CC)rv   c                0   t        j                  | |||ddd      }t        j                  |j	                  d            j                         }t        | ||||      }|j                  d      xs i }t               5 }	|	j                  dt               |t        |j                  d	      xs d
      t        |j                  d      xs |xs d
      t        |d         |d   f       |	j                          d d d        d|d   |dS # 1 sw Y   xY w)N)namepurposedescriptioncharacteristicsFT)r   	sort_keysr   )r   r   r   r   rv   r   zlINSERT INTO product_analyses(created_at,input_hash,provider,model,result_json,ai_run_id) VALUES(?,?,?,?,?,?)r   r?   rv   r~   r   )rx   identificationr   )r   r   hashlibsha256encode	hexdigestr   r   r	   r   r   rN   r   r   )
r   r   r   r   rv   r#   r}   r   r   r   s
             r   product_identificationr      s   

4'^mn  }B  MQ  	RC>>#**W-.88:FT'kZipuvB		2D	 c  CYvc$((:*>*D"&Ec$((SZJ[Jj_dJjhjFklqrtu{r|l}  A  BM  N  O	P

 r&z>>	 s   ?A<DD)r   r   returnrN   )r"   dict[str, Any]r   rN   )r%   rN   r   	list[str])r%   rN   r   r   )r%   rN   rW   r   r   rN   )ip  )r%   rN   r]   rJ   r   r   )ro   list[dict[str, Any]]r   r   )r   rJ   ru   rO   rv   
str | Noner   r   )r   r   )r   rN   r   zdict[str, Any] | None)r   r   )r   rN   r   rN   r   rN   r   r   rv   r   r   r   )!
__future__r   r   r   r(   r   pathlibr   typingr   connectors.ai_providerr   r   r   core_dbr	   core_schemar
   r   source_storer   r   r&   r1   rT   rX   rf   rt   r   r   r   r   r   r5   r   r   <module>r      sw    "   	    ` `  1 %18	( 38t _DD 0{ pt 	?r   