
    ^j/                    |   d dl mZ d dlZd dlZd dlZd dlZd dlmZmZ d dlm	Z	 d dl
mZ d dlmZmZ d dlmZ d dlmZ d	d
dddddddddddddddddddddddddddd d!dd"d#d$dd%d&d'dd(d)d*dgZd;d+Zd<d,Zd=d-Zd=d.Zd>d/Zd?d0Zd@d1Zddd2	 	 	 	 	 dAd3ZdBd4Zdd5dCd6Zd>d7ZdDdEd8Z dFd9Z!dGd:Z"y)H    )annotationsN)datetimetimezone)Path)Any)SOURCE_ROOTconnect)init_core_schema)
configuredTR_TS_004_2011u   ТР ТС 004/2011ud   ТР ТС 004/2011 О безопасности низковольтного оборудования)slugcodequeryTR_TS_005_2011u   ТР ТС 005/2011u?   ТР ТС 005/2011 О безопасности упаковкиTR_TS_010_2011u   ТР ТС 010/2011uU   ТР ТС 010/2011 О безопасности машин и оборудованияTR_TS_012_2011u   ТР ТС 012/2011u:   ТР ТС 012/2011 взрывоопасных средахTR_TS_016_2011u   ТР ТС 016/2011ug   ТР ТС 016/2011 аппаратов работающих на газообразном топливеTR_TS_020_2011u   ТР ТС 020/2011ut   ТР ТС 020/2011 электромагнитная совместимость технических средствTR_TS_032_2013u   ТР ТС 032/2013uZ   ТР ТС 032/2013 оборудование под избыточным давлениемTR_EAEU_037_2016u   ТР ЕАЭС 037/2016uK   ТР ЕАЭС 037/2016 ограничение опасных веществTR_EAEU_043_2017u   ТР ЕАЭС 043/2017uQ   ТР ЕАЭС 043/2017 средства пожарной безопасностиTR_TS_017_2011u   ТР ТС 017/2011uO   ТР ТС 017/2011 продукция легкой промышленности
RF_PP_2425u   ПП РФ № 2425u   Постановление Правительства РФ 2425 обязательная сертификация декларированиеc                     t        j                  t        j                        j	                  d      j                         S )Nr   )microsecond)r   nowr   utcreplace	isoformat     @/var/www/origus_pro_usr/data/www/origus.pro/core/source_store.pyutc_nowr#      s,    <<%--!-<FFHHr!   c                V    t        j                  dd|       j                  d      xs dS )Nz[^A-Za-z0-9_.-]+_document)resubstrip)values    r"   	safe_slugr+       s&    66%sE288=KKr!   c                l    t        j                  dd| j                  dd            j                         S )Nz\s+     )r'   r(   r   r)   texts    r"   normalize_textr1   #   s)    66&#t||FC89??AAr!   c                x    t        j                  t        |       j                  d            j	                         S )Nutf-8)hashlibsha256r1   encode	hexdigestr/   s    r"   content_hashr8   &   s*    >>..55g>?IIKKr!   c            	     .   t                t        j                  dd       t               5 } t        D ]   }| j                  d|d   |d   |d   f       " t        d      r| j                  d       | j                          d d d        y # 1 sw Y   y xY w)	NTparentsexist_okal  
                INSERT INTO tracked_documents(
                    source_key,document_slug,document_code,search_query,enabled
                ) VALUES('techexpert',?,?,?,1)
                ON CONFLICT(source_key,document_slug) DO UPDATE SET
                  document_code=excluded.document_code,
                  search_query=excluded.search_query
            r   r   r   
techexpertzUPDATE connectors SET status=CASE WHEN status='not_configured' THEN 'configured' ELSE status END WHERE connector_key='techexpert')r
   r   mkdirr	   TRACKED_DOCUMENTSexecutesecret_configuredcommit)conitems     r"   init_source_schemarE   )   s    dT2	 c% 	;DKK  fd6l4=9;	; \*KK  \  ]

  s   ABBc                    t                t               5 } | j                  d      j                         }| j                  d      j	                         }| j                  d      j                         d   }d d d        rt        |      nd D cg c]  }t        |       c}t        d      t              dS # 1 sw Y   FxY wc c}w )Na  
            SELECT connector_key AS source_key, connector_name AS source_name,
                   base_url,status,last_test_at AS last_login_at,last_sync_at,last_error,
                   meta_json
            FROM connectors WHERE connector_key='techexpert'
        a  
            SELECT document_slug,document_code,discovered_url,last_checked_at,
                   last_changed_at,last_error,
                   (SELECT COUNT(*) FROM source_snapshots s
                    WHERE s.source_key='techexpert'
                      AND s.document_slug=d.document_slug) AS versions
            FROM tracked_documents d
            WHERE source_key='techexpert' AND enabled=1
            ORDER BY document_code
        zCSELECT COUNT(*) FROM source_snapshots WHERE source_key='techexpert'r   r=   )source	documentsr   snapshots_total)rE   r	   r@   fetchonefetchalldictrA   int)rC   rG   docscountxs        r"   source_statusrQ   :   s    	 oc{{  
 XZ 	 [[ 	 	 XZ 	 kk_`iiklmn#o& #)$v,d&*+T!W+'5u:	 %o o( ,s   A!B;C;Cc                    t        j                  dd|      d d }t        dz  t        |       z  |z  }|j	                  dd       |S )Nz[^0-9]    r=   Tr:   )r'   r(   r   r+   r>   )document_slug
checked_atstampfolders       r"   _snapshot_folderrY   U   sI    
&&2j
)#2
.E|#Im$<<UBF
LLtL,Mr!   )downloaded_namedownloaded_bytesc                \   t                t               }t        |      }	t        | |      }
|
dz  }|
dz  }|
dz  }|j	                  |d       |j	                  |d       d }|r!|r|
t        |      z  }|j                  |       t               5 }|j                  d| f      j                         }| xs |d   |	k7  }|sK|j                  d||| f       |j                          t        j                  |
d	
       d|	|dcd d d        S |j                  d| f       d| |||||	|r|j                  nd d	dd
}|j	                  t        j                  |dd      d       |j                  d| ||||	|t!        |      t!        |      |rt!        |      nd t!        |      t        j                  dd	id      f      }|j                  d||||	|j"                  | f       |j                          d d d        t$        dz  t        |       z  dz  }|j'                         rt        j                  |       |j)                  d	d	       |
j+                         D ]6  }|j-                         st        j.                  |||j                  z         8 d	|	|t!        |
      dS # 1 sw Y   xY w)Nzdocument.txtzdocument.htmlzmanifest.jsonr3   )encodingz
            SELECT last_hash,last_snapshot_id FROM tracked_documents
            WHERE source_key='techexpert' AND document_slug=?
        	last_hashz
                UPDATE tracked_documents SET discovered_url=?,last_checked_at=?,last_error=NULL
                WHERE source_key='techexpert' AND document_slug=?
            Tignore_errorsF)changedhashrV   zZUPDATE source_snapshots SET is_current=0 WHERE source_key='techexpert' AND document_slug=?u   Техэкспертu   TXT является производным представлением. Исходный HTML хранится полностью.)
rG   rU   document_code
source_urltitlerV   r8   downloaded_fileraw_html_is_authoritativetext_extraction_note   )ensure_asciiindentaS  
            INSERT INTO source_snapshots(
              source_key,document_slug,document_code,source_url,checked_at,content_hash,title,
              text_path,html_path,downloaded_file_path,manifest_path,is_current,extraction_status,extraction_meta_json
            ) VALUES('techexpert',?,?,?,?,?,?,?,?,?,?,1,'needs_review',?)
        raw_html_authoritative)rj   z
            UPDATE tracked_documents SET discovered_url=?,last_checked_at=?,last_changed_at=?,
              last_hash=?,last_snapshot_id=?,last_error=NULL
            WHERE source_key='techexpert' AND document_slug=?
        r=   currentr:   )ra   rb   rV   rX   )rE   r#   r8   rY   
write_textr+   write_bytesr	   r@   rJ   rB   shutilrmtreenamejsondumpsstr	lastrowidr   existsr>   iterdiris_filecopy2)rU   rc   rd   re   r0   htmlrZ   r[   rV   digestrX   	text_path	html_pathmanifest_pathdownloaded_pathrC   previousra   manifestcurcurrent_dirsrcs                         r"   save_snapshotr   [   s   yJFM*5F^#I_$I(Mw/w/O+y99##$45	 %c   'hj 	 = 5v =KK  J}57 JJLMM&t4#6zJ% % 	pr  rB  	C+MZg#Ez!OO4H4Hae(, $j
 	  H%q!Q[b cKK 
 =JveCPYN[^_h[i%4S!$s=?QZZ1$7UKMN 	  
:fS]]=Q		S
 	

K%N L(=)AA)KKV]];7dD1~~ A;;=&,,s;sxx+?@A&j#f+VVY% %s   A2J"C%J""J+c                    t                t               }t               5 }|j                  d||d d | f       |j	                          d d d        y # 1 sw Y   y xY w)NzUPDATE tracked_documents SET last_checked_at=?,last_error=?
                       WHERE source_key='techexpert' AND document_slug=?  rE   r#   r	   r@   rB   )rU   errorr   rC   s       r"   set_source_errorr      s[    gi#	 c LMPQVW\X\Q]^kLl	n

  s   )AA)r   c           	         t                t               }t               5 }|j                  d|| rdnd| r| d d nd f       |j	                          d d d        y # 1 sw Y   y xY w)NzrUPDATE connectors SET last_sync_at=?,status=?,last_error=?
                       WHERE connector_key='techexpert'r   okr   r   )r   r   rC   s      r"   set_source_sync_resultr      s_    gi#	 c ;EtEE%4LtT	V 	

	  s   1AA"c                     t                t               } t               5 }|j                  d| f       |j	                          d d d        y # 1 sw Y   y xY w)NzxUPDATE connectors SET last_test_at=?,status='ok',last_error=NULL
                       WHERE connector_key='techexpert'r   )r   rC   s     r"   set_login_okr      sJ    gi#	 c ;<?6	C

  s   $AAc                \   t                d}g }dg}| r"|j                  d       |j                  |        |ddj                  |      z   z  }|dz  }t               5 }|j	                  ||      j                         D cg c]  }t        |       c}cd d d        S c c}w # 1 sw Y   y xY w)Na  
        SELECT s.*,
               kd.status AS knowledge_status,
               kd.completeness_status,
               kd.completeness_score,
               (
                 SELECT COUNT(*)
                 FROM knowledge_rules kr
                 WHERE kr.knowledge_document_id=kd.id
               ) AS knowledge_rules_count
        FROM source_snapshots s
        LEFT JOIN knowledge_documents kd
          ON kd.source_snapshot_id=s.id
    zs.source_key='techexpert'zs.document_slug=?z WHERE z AND z% ORDER BY s.checked_at DESC,s.id DESC)rE   appendjoinr	   r@   rK   rL   )rU   sqlparamswhererC   rP   s         r"   list_snapshotsr      s    	C F&
'E()m$7<<&	&&C	00C	 Ec!$S!8!A!A!CDAQDE EDE Es   #B"?BB"B""B+c                    t                t               5 }|j                  d| f      j                         }|rt	        |      nd cd d d        S # 1 sw Y   y xY w)Nz)SELECT * FROM source_snapshots WHERE id=?)rE   r	   r@   rJ   rL   )snapshot_idrC   rows      r"   get_snapshotr      sK    	 *cKKC[NS\\^tCyT* * *s   0AAc                   t        |       }|st        d      t        |j                  d      xs d      j                  }t               5 }|j                  d| f       |j                  d|d   |d   f      j                         }|j                  d|d   |d   f       |rA|j                  d	|d
   f       |j                  d|d
   |d   |d   |d   |d   |d   f       n|j                  d|d   |d   f       |j                          d d d        |j                         rEt        j                         |j                         j                  v rt        j                  |d       | |d   dS # 1 sw Y   fxY w)Nu!   Версия не найдена.r   rS   z'DELETE FROM source_snapshots WHERE id=?z
            SELECT id,content_hash,checked_at,source_url FROM source_snapshots
            WHERE source_key=? AND document_slug=? ORDER BY checked_at DESC,id DESC LIMIT 1
        
source_keyrU   zOUPDATE source_snapshots SET is_current=0 WHERE source_key=? AND document_slug=?z3UPDATE source_snapshots SET is_current=1 WHERE id=?idzUPDATE tracked_documents SET last_snapshot_id=?,last_hash=?,last_checked_at=?,discovered_url=?
                           WHERE source_key=? AND document_slug=?r8   rV   rd   zUPDATE tracked_documents SET last_snapshot_id=NULL,last_hash=NULL,last_changed_at=NULL
                           WHERE source_key=? AND document_slug=?Tr_   )deletedrU   )r   RuntimeErrorr   getparentr	   r@   rJ   rB   rw   r   resolver;   rp   rq   )r   snaprX   rC   	remainings        r"   delete_snapshotr      s   	k	"D|$GHH)/R077F	 c={nM++  ,_ 568 9A
 	 	e,'_(=>	@KKMyY]N`aKK E"4>)B9\CZ[deq[rl+D,ACD
 KK EFJ<FXY]^mYnEoq

#$ }};..0FNN4D4L4LLf40!$2GHH) s   CE33E<)returnru   )r*   ru   r   ru   )r0   ru   r   ru   )r   None)r   dict[str, Any])rU   ru   rV   ru   r   r   )rU   ru   rc   ru   rd   ru   re   ru   r0   ru   r{   ru   rZ   
str | Noner[   zbytes | Noner   r   )rU   ru   r   ru   r   r   )r   r   r   r   )N)rU   r   r   zlist[dict[str, Any]])r   rM   r   zdict[str, Any] | None)r   rM   r   r   )#
__future__r   r4   rs   r'   rp   r   r   pathlibr   typingr   core_dbr   r	   core_schemar
   secret_storer   rA   r?   r#   r+   r1   r8   rE   rQ   rY   r   r   r   r   r   r   r   r    r!   r"   <module>r      sn   "   	  '   ( ( 8 $8  Bh  i$8  BC  D$8  BY  Z$8A}~$8  Bk  l$8  Bx  y$8  B^  _&>  HU  V&>  H[  \$8  BS  T 4  >N  O ILBL"6 ,0D<W"*<WAK<WUb<W| -1 E4*Ir!   