o DhC@sdZddlZddlZddlZddlmZmZddlmZddl m Z ddl m Z ddl mZddlmZdd lmZmZmZmZmZmZmZmZmZddlZddlZdd lmZdd lm Z d d l!m"Z"m#Z#d dl$m%Z%m&Z&m'Z'm(Z(zddl)m*Z*ej+,dZ-Wn e.ydZ-Ynwzddl/Z/e/0Z1e12e/j3e/j4e12e/j3e/j5dZ6Wn e.ydZ6Ynwe7e8Z9e:ej;j<da=da>da?dedeej@effddZAejBjCddZDde ddZEeEeDd<gdZFhdZGGdddZHd edeeeeIeeIffd!d"ZJ dZd ed#eeeIeIfdeeIeIffd$d%ZKd edejBjLfd&d'ZM d[d ed(eNdeej@effd)d*ZOd+eId(eNd,eNd edeeHf d-d.ZPd+eId/eHd0e#deNfd1d2ZQd+eId/eHd3eNd0e#deeeHeIff d4d5ZRde"dfd+eId(eNd ed0ee#deeIf d6d7ZSddde"d8d+eId3eNd(eNd,eNd edeeHf d9d:ZTd+eIdeNfd;d<ZUd+eIdeNfd=d>ZVd+eIdeNfd?d@ZW     d\dAeeIdBeeeIdCeeIdDeedEeNdFeNdefdGdHZX Id]dDedJeYdeeeIeffdKdLZZ Md^dNeeIdOe[dPeeIgefdQe[deeeIefddff dRdSZ\ dZdNeeIdOe[d0ee#deeeIeIfddffdTdUZ] dZdNeeIdOe[d0ee#deeeIeHfddffdVdWZ^d+eId(eNd,eNd edeeHf dXdYZ_dS)_zG All functions needed to steer and execute downloads of web documents. N)ThreadPoolExecutor as_completed) ConfigParser)partial)version)BytesIO)sleep) AnyCallableDict GeneratorListOptionalSetTupleUnion)UrlStore)redirection_test)DEFAULT_CONFIG Extractor)URL_BLACKLIST_REGEX decode_fileis_acceptable_length make_chunks)SOCKSProxyManager http_proxyTFargsreturncKs8trtntj}tr dtini}d|d<|di||S)zCConfigure urllib3 download pool according to user-defined settings. proxy_url2 num_poolsN) PROXY_URLrurllib3 PoolManager)r manager_class manager_argsr"r"V/home/air/segue/gemini/back/venv/lib/python3.10/site-packages/trafilatura/downloads.py create_poolCsr))accept_encodingz trafilatura/ trafilaturaz( (+https://github.com/adbar/trafilatura) User-Agent)iiiiiiiii i i i i iiiiV> #6:;<@BMRS[c@seZdZdZgdZdedededdfdd Zde fd d Z defd d Z de eefddfddZ de ddfddZde eeffddZdS)Responsez5Store information gathered in a HTTP response object.dataheadershtmlstatusurlr:r=r>rNcCs"||_d|_d|_||_||_dSNr9)selfr:r=r>r"r"r(__init__ls  zResponse.__init__cCs |jduSr?)r:r@r"r"r(__bool__ss zResponse.__bool__cCs|jpt|jSr?)r<rr:rBr"r"r(__repr__vszResponse.__repr__ headerdictcCsdd|D|_dS)z#Store response headers if required.cSsi|] \}}||qSr")lower).0kvr"r"r( |z*Response.store_headers..N)itemsr;)r@rEr"r"r( store_headersyszResponse.store_headersdecodecCs"|r |jrt|j|_dSdSdS)z9Decode the bytestring in data and store a string in html.N)r:rr<)r@rNr"r"r( decode_data~s zResponse.decode_datacsfddjDS)z,Convert the response object to a dictionary.csi|]}|t|qSr")getattr)rGattrrBr"r(rJsz$Response.as_dict..) __slots__rBr"rBr(as_dictszResponse.as_dict)__name__ __module__ __qualname____doc__rRbytesintstrrAboolrCrDr rMrOrSr"r"r"r(r8hsr8configcCs<|jdddd}|r|nd}|ddpd}||fS)zARead and extract HTTP header strings from the configuration file.DEFAULT USER_AGENTS)fallbackNCOOKIE)getstrip splitlines)r\myagents agent_listmycookier"r"r( _parse_configsrhr;cCs>|tkrt|\}}i}|rt||d<|r||d<|ptS)z1Internal function to decide on user-agent string.r,Cookie)rrhrandomchoiceDEFAULT_HEADERS)r\r;rergr"r"r(_determine_headerss rmcCs:tstjj|dd|ddd|dddtdatS)z5Define a retry strategy according to the config file.r] MAX_REDIRECTSrDOWNLOAD_TIMEOUT)totalredirectconnectbackoff_factorstatus_forcelist)RETRY_STRATEGYr$utilRetrygetint FORCE_STATUSr\r"r"r(_get_retry_strategys  r|no_sslcCsN|rtnt}|s%t|dd|rdnt|rdndd}|r#|a|S|a|S)zXCreate a urllib3 pool manager according to options in the config file and HTTPS setting.r]roN CERT_NONE CERT_REQUIRED)timeoutca_certs cert_reqs) NO_CERT_POOL HTTP_POOLr)rycertifiwhere)r\r}poolr"r"r(_initiate_pools   rr> with_headersc CszLt||d}|jd|t|t|dd}t}|dD]}||t||ddkr2t dq| t t ||j |}|rJ||j|WStjjydtd |t|d ||YSty}} ztd || WYd } ~ d Sd } ~ ww) zPInternal function to robustly send a request (SSL or not) and return its result.)r}GETF)r;retriespreload_contentir] MAX_FILE_SIZEzMAX_FILE_SIZE exceededzretrying after SSLError: %sTzdownload error: %s %sN)rrequestrmr| bytearraystreamextendlenry ValueError release_connr8rXr=geturlrMr;r$ exceptionsSSLErrorLOGGERwarning_send_urllib_request Exceptionerror) r>r}rr\ pool_managerresponser:chunkresperrr"r"r(rs8    rroptionscCsDt|jp|jpd}|jdkrtd|j|dSt||s dSdS)z2Check if the response conforms to formal criteria.r_z!not a 200 response: %s for URL %sFT)rr<r:r=rrr)r>rrlentestr"r"r(_is_suitable_responses  rrNcCst|||r |r |jS|SdS)z:Internal function to run safety checks on response result.N)rr<)r>rrNrr"r"r(_handle_responses rcCsL|r|jn|}t|d||d}|r$|jr$|st|d}t|||r$|jSdS)aDownloads a web page and seamlessly decodes the response. Args: url: URL of the page to fetch. no_ssl: Do not try to establish a secure connection (to prevent SSLError). config: Pass configuration values for output control. options: Extraction options (supersedes config). Returns: Unicode string or None in case of failed downloads and invalid results. T)rNr}r\r{N)r\fetch_responser:rrr<)r>r}r\rrr"r"r( fetch_urls   r)rNr}rr\cCsHtstnt}td||||||}|std|dS|||S)aDownloads a web page and returns a full response object. Args: url: URL of the page to fetch. decode: Use html attribute to decode the data (boolean). no_ssl: Don't try to establish a secure connection (to prevent SSLError). with_headers: Keep track of the response headers. config: Pass configuration values for output control. Returns: Response object or None in case of failed downloads and invalid results. zsending request: %szrequest failed: %sN) HAS_PYCURLr_send_pycurl_requestrdebugrO)r>rNr}rr\ dl_functionrr"r"r(r#s    rc Csd}t}|tj|d|tjd|tjd|tjd||jdt r5|tj t z| | |j dk}Wntjy_}ztd||d}WYd}~nd}~ww||S) z+Send a basic HTTP HEAD request with pycurl.Futf-8 rTizpycurl HEAD error: %s %sN)pycurlCurlsetoptURLencodeCONNECTTIMEOUTSSL_VERIFYPEERSSL_VERIFYHOSTNOBODYr# PRE_PROXYperformgetinfo RESPONSE_CODErrrclose)r> page_existscurlrr"r"r(_pycurl_is_live_pageBs&rc CsDzt|}WdSty!}ztd||WYd}~dSd}~ww)zGUse courlan redirection test (based on urllib3) to send a HEAD request.zurllib3 HEAD error: %s %sNFT)rrrr)r>_rr"r"r(_urllib3_is_live_page^s rcCstrt|nd}|p t|S)zCSend a HTTP HEAD request without taking anything else into account.F)rrr)r>resultr"r"r( is_live_pagehs r inputlist blacklist url_filter url_store compressionverbosecs^|dur t|d|d}tt|}rfdd|D}r(fdd|D}|||S)zMFilter, convert input URLs and add them to domain-aware processing dictionaryNF) compressedstrictrcs g|] }td|vr|qS)r_)rsub)rGu)rr"r( ~sz*add_to_compressed_dict..cs&g|]tfddDrqS)c3s|]}|vVqdSr?r")rGfrr"r( sz4add_to_compressed_dict...)any)rG)rrr(rs&)rlistdictfromkeysadd_urls)rrrrrrr")rrr(add_to_compressed_dictos   r@ sleep_timecCs. |j|dd}|s |jr ||fSt|q)zRDetermine threading strategy and draw URLs respecting domain-based back-off rules.Ti) time_limitmax_urls)get_download_urlsdoner)rr bufferlistr"r"r(load_download_buffers r'rdownload_threadsworker chunksizec#svt|d+t||D]}fdd|D}t|D] }|||fVqq WddS1s4wYdS)z3Use a thread pool to perform a series of downloads.) max_workerscsi|] }||qSr")submit)rGr>executorrr"r(rJrKz'_buffered_downloads..N)rrrr)rrrrr future_to_urlfuturer"rr(_buffered_downloadss  "rcCstt|d}t|||S)z3Download queue consumer, single- or multi-threaded.)r)rrr)rrrrr"r"r(buffered_downloadss  rcCs&|r|jnt}tt|d}t|||S)z7Download queue consumer, returns full Response objects.r{)r\rrrr)rrrr\rr"r"r(buffered_response_downloadss  rc Cs*ddt|D}t}|tj|d|tjt|tj ||tj d|tj | dd|tj | dd|tj| dd|tj| dd|tjd|d urt|tjd |tjd n |tjt|rt}|tj|jtr|tjtz|}Wn:tjy}z-td |||d ur|jd tvrt d ||t!|d ||WYd}~SWYd}~dSd}~wwt"||#|j$|#|j%} |&|ri} |'j(ddd)D]} d| vrq| *dd\} } | +| | +<q| ,| | S)zDExperimental function using libcurl and pycurl to speed up downloadscSsg|] \}}|d|qS)z: r")rGheadercontentr"r"r(rsz(_send_pycurl_request..rrr]rnrorTrzpycurl error: %s %sFzretrying after SSL error: %s %sNz iso-8859-1replace)errors:)-rmrLrrrrrSHARE CURL_SHARE HTTPHEADERFOLLOWLOCATION MAXREDIRSryrTIMEOUT MAXFILESIZENOSIGNALrrCAINFOrrrHEADERFUNCTIONwriter#r perform_rbrrrCURL_SSL_ERRORSrrr8rr EFFECTIVE_URLrgetvaluerNrdsplitrcrM)r>r}rr\ headerlistr headerbytes bufferbytesrr respheaderslinenamevaluer"r"r(rs\   rr?)F)NNNFF)r)r)`rWloggingosrjconcurrent.futuresrr configparserr functoolsrimportlib.metadatariortimertypingr r r r r rrrrrr$courlanrcourlan.networkrsettingsrrutilsrrrrurllib3.contrib.socksrenvironrbr# ImportErrorr CurlSharerrSH_SHARE LOCK_DATA_DNSLOCK_DATA_SSL_SESSIONr getLoggerrTrdisable_warningsrInsecureRequestWarningrrrvr%r)rw make_headersrl USER_AGENTrzrr8rZrhrmrxr|r[rrrrrrrrrrfloatrrYrrrrr"r"r"r(s~     ,       &"      %