derek/AI-Stock-Trader
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Import python venv for stability

Browse Source
This commit is contained in:
Derek Holloway
2026-02-15 21:24:16 -08:00
parent 1343e93a59
commit 7d784705c9
4997 changed files with 1628270 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
WebServer/AIPython/python/lib/python3.11/site-packages/curl_cffi/__init__.py
+96
View File
@@ -0,0 +1,96 @@
__all__ = [
"Curl",
"AsyncCurl",
"CurlMime",
"CurlError",
"CurlInfo",
"CurlOpt",
"CurlMOpt",
"CurlECode",
"CurlHttpVersion",
"CurlSslVersion",
"CurlWsFlag",
"config_warnings",
"ffi",
"is_pro",
"lib",
"Session",
"AsyncSession",
"BrowserType",
"BrowserTypeLiteral",
"request",
"head",
"get",
"post",
"put",
"patch",
"delete",
"options",
"Cookies",
"Headers",
"Request",
"Response",
"AsyncWebSocket",
"WebSocket",
"WebSocketError",
"WebSocketClosed",
"WebSocketTimeout",
"WsCloseCode",
"ExtraFingerprints",
"CookieTypes",
"HeaderTypes",
"ProxySpec",
"exceptions",
]
import _cffi_backend # noqa: F401 # required by _wrapper
from .__version__ import __curl_version__, __description__, __title__, __version__ # noqa: F401
# This line includes _wrapper.so into the wheel
from ._wrapper import ffi, lib
from .aio import AsyncCurl
from .const import (
CurlECode,
CurlHttpVersion,
CurlInfo,
CurlMOpt,
CurlOpt,
CurlSslVersion,
CurlWsFlag,
)
from .curl import Curl, CurlError, CurlMime
from .requests import (
AsyncSession,
AsyncWebSocket,
BrowserType,
BrowserTypeLiteral,
Cookies,
CookieTypes,
ExtraFingerprints,
Headers,
HeaderTypes,
ProxySpec,
Request,
Response,
Session,
WebSocket,
WebSocketClosed,
WebSocketError,
WebSocketTimeout,
WsCloseCode,
delete,
exceptions,
get,
head,
options,
patch,
post,
put,
request,
)
from .utils import config_warnings, is_pro
config_warnings(on=False)
WebServer/AIPython/python/lib/python3.11/site-packages/curl_cffi/__version__.py
+8
View File
@@ -0,0 +1,8 @@
from importlib import metadata
from .curl import Curl
__title__ = "curl_cffi"
__description__ = metadata.metadata("curl_cffi")["Summary"]
__version__ = metadata.version("curl_cffi")
__curl_version__ = Curl().version().decode()
WebServer/AIPython/python/lib/python3.11/site-packages/curl_cffi/_asyncio_selector.py
+344
View File
@@ -0,0 +1,344 @@
"""Ensure asyncio selector methods (add_reader, etc.) are available
tornado 6.1 adds AddThreadSelectorEventLoop event loop,
running select in a thread and defining these methods on the running event loop.
This factors out the functionality of AddThreadSelectorEventLoop
into a standalone SelectorThread object which can be attached to any running event loop.
Vendored from tornado v6.4.0
Redistributed under license Apache-2.0
"""
import asyncio
import atexit
import errno
import functools
import select
import socket
import threading
import typing
from contextlib import suppress
from typing import (
Any,
Callable,
Optional,
Protocol,
TypeVar,
Union,
)
_T = TypeVar("_T")
class _HasFileno(Protocol):
def fileno(self) -> int:
return 0
_FileDescriptorLike = Union[int, _HasFileno]
# Collection of selector thread event loops to shut down on exit.
_selector_loops: set["SelectorThread"] = set()
def _atexit_callback() -> None:
for loop in _selector_loops:
with loop._select_cond:
loop._closing_selector = True
loop._select_cond.notify()
with suppress(BlockingIOError):
loop._waker_w.send(b"a")
if loop._thread is not None:
# If we don't join our (daemon) thread here, we may get a deadlock
# during interpreter shutdown. I don't really understand why. This
# deadlock happens every time in CI (both travis and appveyor) but
# I've never been able to reproduce locally.
loop._thread.join()
_selector_loops.clear()
atexit.register(_atexit_callback)
class SelectorThread:
"""Define ``add_reader`` methods to be called in a background select thread.
Instances of this class start a second thread to run a selector.
This thread is completely hidden from the user;
all callbacks are run on the wrapped event loop's thread.
Typically used via ``AddThreadSelectorEventLoop``,
but can be attached to a running asyncio loop.
"""
_closed = False
def __init__(self, real_loop: asyncio.AbstractEventLoop) -> None:
self._real_loop = real_loop
self._select_cond = threading.Condition()
self._select_args: Optional[
tuple[list[_FileDescriptorLike], list[_FileDescriptorLike]]
] = None
self._closing_selector = False
self._thread: Optional[threading.Thread] = None
self._thread_manager_handle = self._thread_manager()
async def thread_manager_anext() -> None:
# the anext builtin wasn't added until 3.10. We just need to iterate
# this generator one step.
await self._thread_manager_handle.__anext__()
# When the loop starts, start the thread. Not too soon because we can't
# clean up if we get to this point but the event loop is closed without
# starting.
self._real_loop.call_soon(
lambda: self._real_loop.create_task(thread_manager_anext())
)
self._readers: dict[_FileDescriptorLike, Callable] = {}
self._writers: dict[_FileDescriptorLike, Callable] = {}
# Writing to _waker_w will wake up the selector thread, which
# watches for _waker_r to be readable.
self._waker_r, self._waker_w = socket.socketpair()
self._waker_r.setblocking(False)
self._waker_w.setblocking(False)
_selector_loops.add(self)
self.add_reader(self._waker_r, self._consume_waker)
def close(self) -> None:
if self._closed:
return
with self._select_cond:
self._closing_selector = True
self._select_cond.notify()
self._wake_selector()
if self._thread is not None:
self._thread.join()
_selector_loops.discard(self)
self.remove_reader(self._waker_r)
self._waker_r.close()
self._waker_w.close()
self._closed = True
async def _thread_manager(self) -> typing.AsyncGenerator[None, None]:
# Create a thread to run the select system call. We manage this thread
# manually so we can trigger a clean shutdown from an atexit hook. Note
# that due to the order of operations at shutdown, only daemon threads
# can be shut down in this way (non-daemon threads would require the
# introduction of a new hook: https://bugs.python.org/issue41962)
self._thread = threading.Thread(
name="Tornado selector",
daemon=True,
target=self._run_select,
)
self._thread.start()
self._start_select()
try:
# The presense of this yield statement means that this coroutine
# is actually an asynchronous generator, which has a special
# shutdown protocol. We wait at this yield point until the
# event loop's shutdown_asyncgens method is called, at which point
# we will get a GeneratorExit exception and can shut down the
# selector thread.
yield
except GeneratorExit:
self.close()
raise
def _wake_selector(self) -> None:
if self._closed:
return
with suppress(BlockingIOError):
self._waker_w.send(b"a")
def _consume_waker(self) -> None:
with suppress(BlockingIOError):
self._waker_r.recv(1024)
def _start_select(self) -> None:
# Capture reader and writer sets here in the event loop
# thread to avoid any problems with concurrent
# modification while the select loop uses them.
with self._select_cond:
assert self._select_args is None
self._select_args = (list(self._readers.keys()), list(self._writers.keys()))
self._select_cond.notify()
def _run_select(self) -> None:
while True:
with self._select_cond:
while self._select_args is None and not self._closing_selector:
self._select_cond.wait()
if self._closing_selector:
return
assert self._select_args is not None
to_read, to_write = self._select_args
self._select_args = None
# We use the simpler interface of the select module instead of
# the more stateful interface in the selectors module because
# this class is only intended for use on windows, where
# select.select is the only option. The selector interface
# does not have well-documented thread-safety semantics that
# we can rely on so ensuring proper synchronization would be
# tricky.
try:
# On windows, selecting on a socket for write will not
# return the socket when there is an error (but selecting
# for reads works). Also select for errors when selecting
# for writes, and merge the results.
#
# This pattern is also used in
# https://github.com/python/cpython/blob/v3.8.0/Lib/selectors.py#L312-L317
rs, ws, xs = select.select(to_read, to_write, to_write)
ws = ws + xs
except OSError as e:
# After remove_reader or remove_writer is called, the file
# descriptor may subsequently be closed on the event loop
# thread. It's possible that this select thread hasn't
# gotten into the select system call by the time that
# happens in which case (at least on macOS), select may
# raise a "bad file descriptor" error. If we get that
# error, check and see if we're also being woken up by
# polling the waker alone. If we are, just return to the
# event loop and we'll get the updated set of file
# descriptors on the next iteration. Otherwise, raise the
# original error.
if e.errno == getattr(errno, "WSAENOTSOCK", errno.EBADF):
rs, _, _ = select.select([self._waker_r.fileno()], [], [], 0)
if rs:
ws = []
else:
raise
else:
raise
try:
self._real_loop.call_soon_threadsafe(self._handle_select, rs, ws)
except RuntimeError:
# "Event loop is closed". Swallow the exception for
# consistency with PollIOLoop (and logical consistency
# with the fact that we can't guarantee that an
# add_callback that completes without error will
# eventually execute).
pass
except AttributeError:
# ProactorEventLoop may raise this instead of RuntimeError
# if call_soon_threadsafe races with a call to close().
# Swallow it too for consistency.
pass
def _handle_select(
self, rs: list[_FileDescriptorLike], ws: list[_FileDescriptorLike]
) -> None:
for r in rs:
self._handle_event(r, self._readers)
for w in ws:
self._handle_event(w, self._writers)
self._start_select()
def _handle_event(
self,
fd: _FileDescriptorLike,
cb_map: dict[_FileDescriptorLike, Callable],
) -> None:
try:
callback = cb_map[fd]
except KeyError:
return
callback()
def add_reader(
self, fd: _FileDescriptorLike, callback: Callable[..., None], *args: Any
) -> None:
self._readers[fd] = functools.partial(callback, *args)
self._wake_selector()
def add_writer(
self, fd: _FileDescriptorLike, callback: Callable[..., None], *args: Any
) -> None:
self._writers[fd] = functools.partial(callback, *args)
self._wake_selector()
def remove_reader(self, fd: _FileDescriptorLike) -> bool:
try:
del self._readers[fd]
except KeyError:
return False
self._wake_selector()
return True
def remove_writer(self, fd: _FileDescriptorLike) -> bool:
try:
del self._writers[fd]
except KeyError:
return False
self._wake_selector()
return True
class AddThreadSelectorEventLoop(asyncio.AbstractEventLoop):
"""Wrap an event loop to add implementations of the ``add_reader`` method family.
Instances of this class start a second thread to run a selector.
This thread is completely hidden from the user; all callbacks are
run on the wrapped event loop's thread.
This class is used automatically by Tornado; applications should not need
to refer to it directly.
It is safe to wrap any event loop with this class, although it only makes sense
for event loops that do not implement the ``add_reader`` family of methods
themselves (i.e. ``WindowsProactorEventLoop``)
Closing the ``AddThreadSelectorEventLoop`` also closes the wrapped event loop.
"""
# This class is a __getattribute__-based proxy. All attributes other than those
# in this set are proxied through to the underlying loop.
MY_ATTRIBUTES = {
"_real_loop",
"_selector",
"add_reader",
"add_writer",
"close",
"remove_reader",
"remove_writer",
}
def __getattribute__(self, name: str) -> Any:
if name in AddThreadSelectorEventLoop.MY_ATTRIBUTES:
return super().__getattribute__(name)
return getattr(self._real_loop, name)
def __init__(self, real_loop: asyncio.AbstractEventLoop) -> None:
self._real_loop = real_loop
self._selector = SelectorThread(real_loop)
def close(self) -> None:
self._selector.close()
self._real_loop.close()
def add_reader( # type: ignore
self,
fd: "_FileDescriptorLike",
callback: Callable[..., None],
*args: Any,
) -> None:
return self._selector.add_reader(fd, callback, *args)
def add_writer( # type: ignore
self,
fd: "_FileDescriptorLike",
callback: Callable[..., None],
*args: Any, # type: ignore
) -> None:
return self._selector.add_writer(fd, callback, *args)
def remove_reader(self, fd: "_FileDescriptorLike") -> bool:
return self._selector.remove_reader(fd)
def remove_writer(self, fd: "_FileDescriptorLike") -> bool:
return self._selector.remove_writer(fd)
WebServer/AIPython/python/lib/python3.11/site-packages/curl_cffi/_wrapper.abi3.so
Executable
BIN
View File
Binary file not shown.
WebServer/AIPython/python/lib/python3.11/site-packages/curl_cffi/aio.py
+343
View File
@@ -0,0 +1,343 @@
import asyncio
import sys
import warnings
from contextlib import suppress
from typing import Any, Optional
from weakref import WeakKeyDictionary
from ._wrapper import ffi, lib
from .const import CurlECode, CurlMOpt
from .curl import DEFAULT_CACERT, Curl, CurlError
from .utils import CurlCffiWarning
__all__ = ["AsyncCurl"]
if sys.platform == "win32":
# registry of asyncio loop : selector thread
_selectors: WeakKeyDictionary = WeakKeyDictionary()
PROACTOR_WARNING = """
Proactor event loop does not implement add_reader family of methods required.
Registering an additional selector thread for add_reader support.
To avoid this warning use:
asyncio.set_event_loop_policy(WindowsSelectorEventLoopPolicy())
"""
def get_selector(
asyncio_loop: asyncio.AbstractEventLoop,
) -> asyncio.AbstractEventLoop:
"""Get selector-compatible loop
Returns an object with ``add_reader`` family of methods,
either the loop itself or a SelectorThread instance.
Workaround Windows proactor removal of *reader methods.
"""
if asyncio_loop in _selectors:
return _selectors[asyncio_loop]
if not isinstance(
asyncio_loop, getattr(asyncio, "ProactorEventLoop", type(None))
):
return asyncio_loop
warnings.warn(PROACTOR_WARNING, CurlCffiWarning, stacklevel=2)
from ._asyncio_selector import AddThreadSelectorEventLoop
selector_loop = _selectors[asyncio_loop] = AddThreadSelectorEventLoop(
asyncio_loop
) # type: ignore
# patch loop.close to also close the selector thread
loop_close = asyncio_loop.close
def _close_selector_and_loop():
# restore original before calling selector.close,
# which in turn calls eventloop.close!
asyncio_loop.close = loop_close
_selectors.pop(asyncio_loop, None)
selector_loop.close()
asyncio_loop.close = _close_selector_and_loop
return selector_loop
else:
def get_selector(loop: asyncio.AbstractEventLoop) -> asyncio.AbstractEventLoop:
return loop
CURL_POLL_NONE = 0
CURL_POLL_IN = 1
CURL_POLL_OUT = 2
CURL_POLL_INOUT = 3
CURL_POLL_REMOVE = 4
CURL_SOCKET_TIMEOUT = -1
CURL_SOCKET_BAD = -1
CURL_CSELECT_IN = 0x01
CURL_CSELECT_OUT = 0x02
CURL_CSELECT_ERR = 0x04
CURLMSG_DONE = 1
CURLPIPE_NOTHING = 0
CURLPIPE_HTTP1 = 1 # deprecated
CURLPIPE_MULTIPLEX = 2
"""
libcurl provides an event-based system for multiple handles with the following API:
- curl_multi_socket_action, for detecting events
- curl_multi_info_read, for reading the transfer status
There are 2 callbacks:
- socket_function, set by CURLMOPT_SOCKETFUNCTION, will be called for socket events.
- timer_function, set by CURLMOPT_TIMERFUNCTION, will be called when timeouts happen.
And it works like the following:
Set up handles, callbacks first.
When started, curl_multi_socket_action should be called to start everything.
If there are data in/out, libcurl calls the socket_function callback, and it sets up
`process_data` as asyncio loop reader/writer function. `process_data` will call
curl_multi_info_read to determine whether a certain `await perform` has finished.
When idle, libcurl will call the timer_function callback, which sets up a later call
for socket_action to detect events.
"""
@ffi.def_extern()
def timer_function(curlm, timeout_ms: int, clientp: Any) -> int:
"""
see: https://curl.se/libcurl/c/CURLMOPT_TIMERFUNCTION.html
"""
async_curl = ffi.from_handle(clientp)
# Cancel the timer anyway, if it's -1, yes, libcurl says it should be cancelled.
# If not, to add a new timer, we need to cancel the old timer.
if async_curl._timer:
async_curl._timer.cancel() # If already called, cancel does nothing.
async_curl._timer = None
# libcurl says to install a timer which calls socket_action on fire.
async_curl._timer = async_curl.loop.call_later(
timeout_ms / 1000,
async_curl.process_data,
CURL_SOCKET_TIMEOUT, # -1
CURL_POLL_NONE, # 0
)
return 0
@ffi.def_extern()
def socket_function(curl, sockfd: int, what: int, clientp: Any, data: Any) -> int:
"""This callback is called when libcurl decides it's time to interact with certain
sockets"""
async_curl = ffi.from_handle(clientp)
loop = async_curl.loop
# Always remove and re-add fds
if sockfd in async_curl._sockfds:
loop.remove_reader(sockfd)
loop.remove_writer(sockfd)
# Need to read from the socket
if what & CURL_POLL_IN:
loop.add_reader(sockfd, async_curl.process_data, sockfd, CURL_CSELECT_IN)
async_curl._sockfds.add(sockfd)
# Need to write to the socket
if what & CURL_POLL_OUT:
loop.add_writer(sockfd, async_curl.process_data, sockfd, CURL_CSELECT_OUT)
async_curl._sockfds.add(sockfd)
# Need to remove the socket
if what == CURL_POLL_REMOVE:
async_curl._sockfds.remove(sockfd)
return 0
class AsyncCurl:
"""Wrapper around curl_multi handle to provide asyncio support. It uses the libcurl
socket_action APIs."""
def __init__(self, cacert: str = "", loop=None):
"""
Parameters:
cacert: CA cert path to use, by default, certs from ``certifi`` are used.
loop: EventLoop to use.
"""
self._curlm = lib.curl_multi_init()
self._cacert = cacert or DEFAULT_CACERT
self._curl2future: dict[Curl, asyncio.Future] = {} # curl to future map
self._curl2curl: dict[ffi.CData, Curl] = {} # c curl to Curl
self._sockfds: set[int] = set() # sockfds
self.loop = get_selector(
loop if loop is not None else asyncio.get_running_loop()
)
self._timeout_checker = self.loop.create_task(self._force_timeout())
self._timer: Optional[asyncio.TimerHandle] = None
self._setup()
def _setup(self):
self.setopt(CurlMOpt.TIMERFUNCTION, lib.timer_function)
self.setopt(CurlMOpt.SOCKETFUNCTION, lib.socket_function)
self._self_handle = ffi.new_handle(self)
self.setopt(CurlMOpt.SOCKETDATA, self._self_handle)
self.setopt(CurlMOpt.TIMERDATA, self._self_handle)
# self.setopt(CurlMOpt.PIPELINING, CURLPIPE_NOTHING)
async def close(self):
"""Close and cleanup running timers, readers, writers and handles."""
# Close and wait for the force timeout checker to complete
self._timeout_checker.cancel()
with suppress(asyncio.CancelledError):
await self._timeout_checker
# Close all pending futures
for curl, future in self._curl2future.items():
lib.curl_multi_remove_handle(self._curlm, curl._curl)
if not future.done() and not future.cancelled():
future.set_result(None)
# Cleanup curl_multi handle
lib.curl_multi_cleanup(self._curlm)
self._curlm = None
# Remove add readers and writers
for sockfd in self._sockfds:
self.loop.remove_reader(sockfd)
self.loop.remove_writer(sockfd)
# Cancel all time functions
if self._timer:
self._timer.cancel()
async def _force_timeout(self):
"""This coroutine is used to safeguard from any missing signals from curl, and
put everything back on track"""
while True:
if not self._curlm:
break
self.socket_action(CURL_SOCKET_TIMEOUT, CURL_POLL_NONE)
await asyncio.sleep(0.1)
def add_handle(self, curl: Curl):
"""Add a curl handle to be managed by curl_multi. This is the equivalent of
`perform` in the async world."""
curl._ensure_cacert()
errcode = lib.curl_multi_add_handle(self._curlm, curl._curl)
self._check_error(errcode)
future = self.loop.create_future()
self._curl2future[curl] = future
self._curl2curl[curl._curl] = curl
return future
def socket_action(self, sockfd: int, ev_bitmask: int) -> int:
"""wrapper for curl_multi_socket_action,
returns the number of running curl handles."""
running_handle = ffi.new("int *")
errcode = lib.curl_multi_socket_action(
self._curlm, sockfd, ev_bitmask, running_handle
)
self._check_error(errcode)
return running_handle[0]
def process_data(self, sockfd: int, ev_bitmask: int):
"""Call curl_multi_info_read to read data for given socket."""
if not self._curlm:
warnings.warn(
"Curlm already closed! quitting from process_data",
CurlCffiWarning,
stacklevel=2,
)
return
self.socket_action(sockfd, ev_bitmask)
msg_in_queue = ffi.new("int *")
while True:
try:
curl_msg = lib.curl_multi_info_read(self._curlm, msg_in_queue)
# NULL is returned as a signal that no more to be get at this point
if curl_msg == ffi.NULL:
break
if curl_msg.msg == CURLMSG_DONE:
curl = self._curl2curl[curl_msg.easy_handle]
retcode = curl_msg.data.result
if retcode == 0:
self.set_result(curl)
else:
self.set_exception(curl, curl._get_error(retcode, "perform"))
else:
print("NOT DONE") # Will not reach, for nothing else being defined.
except Exception:
warnings.warn(
"Unexpected curl multi state in process_data, "
"please open an issue on GitHub\n",
CurlCffiWarning,
stacklevel=2,
)
def _pop_future(self, curl: Curl):
errcode = lib.curl_multi_remove_handle(self._curlm, curl._curl)
self._check_error(errcode)
self._curl2curl.pop(curl._curl, None)
return self._curl2future.pop(curl, None)
def remove_handle(self, curl: Curl):
"""Cancel a future for given curl handle."""
future = self._pop_future(curl)
if future and not future.done() and not future.cancelled():
future.cancel()
def set_result(self, curl: Curl):
"""Mark a future as done for given curl handle."""
future = self._pop_future(curl)
if future and not future.done() and not future.cancelled():
future.set_result(None)
def set_exception(self, curl: Curl, exception):
"""Raise exception of a future for given curl handle."""
future = self._pop_future(curl)
if future and not future.done() and not future.cancelled():
future.set_exception(exception)
def _check_error(self, errcode: int, *args: Any):
if errcode == CurlECode.OK:
return
errmsg = lib.curl_multi_strerror(errcode)
action = " ".join([str(a) for a in args])
raise CurlError(
f"Failed in {action}, multi: ({errcode}) {errmsg}. "
"See https://curl.se/libcurl/c/libcurl-errors.html first for more "
"details. Please open an issue on GitHub to help debug this error.",
)
def setopt(self, option, value):
"""Wrapper around curl_multi_setopt."""
if option in (
CurlMOpt.PIPELINING,
CurlMOpt.MAXCONNECTS,
CurlMOpt.MAX_HOST_CONNECTIONS,
CurlMOpt.MAX_PIPELINE_LENGTH,
CurlMOpt.MAX_TOTAL_CONNECTIONS,
CurlMOpt.MAX_CONCURRENT_STREAMS,
):
c_value = ffi.new("long*", value)
else:
c_value = value
return lib.curl_multi_setopt(self._curlm, option, c_value)
WebServer/AIPython/python/lib/python3.11/site-packages/curl_cffi/const.py
+613
View File
@@ -0,0 +1,613 @@
# This file is automatically generated, do not modify it directly.
from enum import IntEnum
class CurlOpt(IntEnum):
"""``CULROPT_`` constancs extracted from libcurl,
see: https://curl.se/libcurl/c/curl_easy_setopt.html"""
WRITEDATA = 10000 + 1
URL = 10000 + 2
PORT = 0 + 3
PROXY = 10000 + 4
USERPWD = 10000 + 5
PROXYUSERPWD = 10000 + 6
RANGE = 10000 + 7
READDATA = 10000 + 9
ERRORBUFFER = 10000 + 10
WRITEFUNCTION = 20000 + 11
READFUNCTION = 20000 + 12
TIMEOUT = 0 + 13
INFILESIZE = 0 + 14
POSTFIELDS = 10000 + 15
REFERER = 10000 + 16
FTPPORT = 10000 + 17
USERAGENT = 10000 + 18
LOW_SPEED_LIMIT = 0 + 19
LOW_SPEED_TIME = 0 + 20
RESUME_FROM = 0 + 21
COOKIE = 10000 + 22
HTTPHEADER = 10000 + 23
HTTPPOST = 10000 + 24
SSLCERT = 10000 + 25
KEYPASSWD = 10000 + 26
CRLF = 0 + 27
QUOTE = 10000 + 28
HEADERDATA = 10000 + 29
COOKIEFILE = 10000 + 31
SSLVERSION = 0 + 32
TIMECONDITION = 0 + 33
TIMEVALUE = 0 + 34
CUSTOMREQUEST = 10000 + 36
STDERR = 10000 + 37
POSTQUOTE = 10000 + 39
VERBOSE = 0 + 41
HEADER = 0 + 42
NOPROGRESS = 0 + 43
NOBODY = 0 + 44
FAILONERROR = 0 + 45
UPLOAD = 0 + 46
POST = 0 + 47
DIRLISTONLY = 0 + 48
APPEND = 0 + 50
NETRC = 0 + 51
FOLLOWLOCATION = 0 + 52
TRANSFERTEXT = 0 + 53
PUT = 0 + 54
PROGRESSFUNCTION = 20000 + 56
XFERINFODATA = 10000 + 57
AUTOREFERER = 0 + 58
PROXYPORT = 0 + 59
POSTFIELDSIZE = 0 + 60
HTTPPROXYTUNNEL = 0 + 61
INTERFACE = 10000 + 62
KRBLEVEL = 10000 + 63
SSL_VERIFYPEER = 0 + 64
CAINFO = 10000 + 65
MAXREDIRS = 0 + 68
FILETIME = 0 + 69
TELNETOPTIONS = 10000 + 70
MAXCONNECTS = 0 + 71
FRESH_CONNECT = 0 + 74
FORBID_REUSE = 0 + 75
RANDOM_FILE = 10000 + 76
EGDSOCKET = 10000 + 77
CONNECTTIMEOUT = 0 + 78
HEADERFUNCTION = 20000 + 79
HTTPGET = 0 + 80
SSL_VERIFYHOST = 0 + 81
COOKIEJAR = 10000 + 82
SSL_CIPHER_LIST = 10000 + 83
HTTP_VERSION = 0 + 84
FTP_USE_EPSV = 0 + 85
SSLCERTTYPE = 10000 + 86
SSLKEY = 10000 + 87
SSLKEYTYPE = 10000 + 88
SSLENGINE = 10000 + 89
SSLENGINE_DEFAULT = 0 + 90
DNS_USE_GLOBAL_CACHE = 0 + 91
DNS_CACHE_TIMEOUT = 0 + 92
PREQUOTE = 10000 + 93
DEBUGFUNCTION = 20000 + 94
DEBUGDATA = 10000 + 95
COOKIESESSION = 0 + 96
CAPATH = 10000 + 97
BUFFERSIZE = 0 + 98
NOSIGNAL = 0 + 99
SHARE = 10000 + 100
PROXYTYPE = 0 + 101
ACCEPT_ENCODING = 10000 + 102
PRIVATE = 10000 + 103
HTTP200ALIASES = 10000 + 104
UNRESTRICTED_AUTH = 0 + 105
FTP_USE_EPRT = 0 + 106
HTTPAUTH = 0 + 107
SSL_CTX_FUNCTION = 20000 + 108
SSL_CTX_DATA = 10000 + 109
FTP_CREATE_MISSING_DIRS = 0 + 110
PROXYAUTH = 0 + 111
SERVER_RESPONSE_TIMEOUT = 0 + 112
IPRESOLVE = 0 + 113
MAXFILESIZE = 0 + 114
INFILESIZE_LARGE = 30000 + 115
RESUME_FROM_LARGE = 30000 + 116
MAXFILESIZE_LARGE = 30000 + 117
NETRC_FILE = 10000 + 118
USE_SSL = 0 + 119
POSTFIELDSIZE_LARGE = 30000 + 120
TCP_NODELAY = 0 + 121
FTPSSLAUTH = 0 + 129
IOCTLFUNCTION = 20000 + 130
IOCTLDATA = 10000 + 131
FTP_ACCOUNT = 10000 + 134
COOKIELIST = 10000 + 135
IGNORE_CONTENT_LENGTH = 0 + 136
FTP_SKIP_PASV_IP = 0 + 137
FTP_FILEMETHOD = 0 + 138
LOCALPORT = 0 + 139
LOCALPORTRANGE = 0 + 140
CONNECT_ONLY = 0 + 141
CONV_FROM_NETWORK_FUNCTION = 20000 + 142
CONV_TO_NETWORK_FUNCTION = 20000 + 143
CONV_FROM_UTF8_FUNCTION = 20000 + 144
MAX_SEND_SPEED_LARGE = 30000 + 145
MAX_RECV_SPEED_LARGE = 30000 + 146
FTP_ALTERNATIVE_TO_USER = 10000 + 147
SOCKOPTFUNCTION = 20000 + 148
SOCKOPTDATA = 10000 + 149
SSL_SESSIONID_CACHE = 0 + 150
SSH_AUTH_TYPES = 0 + 151
SSH_PUBLIC_KEYFILE = 10000 + 152
SSH_PRIVATE_KEYFILE = 10000 + 153
FTP_SSL_CCC = 0 + 154
TIMEOUT_MS = 0 + 155
CONNECTTIMEOUT_MS = 0 + 156
HTTP_TRANSFER_DECODING = 0 + 157
HTTP_CONTENT_DECODING = 0 + 158
NEW_FILE_PERMS = 0 + 159
NEW_DIRECTORY_PERMS = 0 + 160
POSTREDIR = 0 + 161
SSH_HOST_PUBLIC_KEY_MD5 = 10000 + 162
OPENSOCKETFUNCTION = 20000 + 163
OPENSOCKETDATA = 10000 + 164
COPYPOSTFIELDS = 10000 + 165
PROXY_TRANSFER_MODE = 0 + 166
SEEKFUNCTION = 20000 + 167
SEEKDATA = 10000 + 168
CRLFILE = 10000 + 169
ISSUERCERT = 10000 + 170
ADDRESS_SCOPE = 0 + 171
CERTINFO = 0 + 172
USERNAME = 10000 + 173
PASSWORD = 10000 + 174
PROXYUSERNAME = 10000 + 175
PROXYPASSWORD = 10000 + 176
NOPROXY = 10000 + 177
TFTP_BLKSIZE = 0 + 178
SOCKS5_GSSAPI_SERVICE = 10000 + 179
SOCKS5_GSSAPI_NEC = 0 + 180
PROTOCOLS = 0 + 181
REDIR_PROTOCOLS = 0 + 182
SSH_KNOWNHOSTS = 10000 + 183
SSH_KEYFUNCTION = 20000 + 184
SSH_KEYDATA = 10000 + 185
MAIL_FROM = 10000 + 186
MAIL_RCPT = 10000 + 187
FTP_USE_PRET = 0 + 188
RTSP_REQUEST = 0 + 189
RTSP_SESSION_ID = 10000 + 190
RTSP_STREAM_URI = 10000 + 191
RTSP_TRANSPORT = 10000 + 192
RTSP_CLIENT_CSEQ = 0 + 193
RTSP_SERVER_CSEQ = 0 + 194
INTERLEAVEDATA = 10000 + 195
INTERLEAVEFUNCTION = 20000 + 196
WILDCARDMATCH = 0 + 197
CHUNK_BGN_FUNCTION = 20000 + 198
CHUNK_END_FUNCTION = 20000 + 199
FNMATCH_FUNCTION = 20000 + 200
CHUNK_DATA = 10000 + 201
FNMATCH_DATA = 10000 + 202
RESOLVE = 10000 + 203
TLSAUTH_USERNAME = 10000 + 204
TLSAUTH_PASSWORD = 10000 + 205
TLSAUTH_TYPE = 10000 + 206
TRANSFER_ENCODING = 0 + 207
CLOSESOCKETFUNCTION = 20000 + 208
CLOSESOCKETDATA = 10000 + 209
GSSAPI_DELEGATION = 0 + 210
DNS_SERVERS = 10000 + 211
ACCEPTTIMEOUT_MS = 0 + 212
TCP_KEEPALIVE = 0 + 213
TCP_KEEPIDLE = 0 + 214
TCP_KEEPINTVL = 0 + 215
SSL_OPTIONS = 0 + 216
MAIL_AUTH = 10000 + 217
SASL_IR = 0 + 218
XFERINFOFUNCTION = 20000 + 219
XOAUTH2_BEARER = 10000 + 220
DNS_INTERFACE = 10000 + 221
DNS_LOCAL_IP4 = 10000 + 222
DNS_LOCAL_IP6 = 10000 + 223
LOGIN_OPTIONS = 10000 + 224
SSL_ENABLE_NPN = 0 + 225
SSL_ENABLE_ALPN = 0 + 226
EXPECT_100_TIMEOUT_MS = 0 + 227
PROXYHEADER = 10000 + 228
HEADEROPT = 0 + 229
PINNEDPUBLICKEY = 10000 + 230
UNIX_SOCKET_PATH = 10000 + 231
SSL_VERIFYSTATUS = 0 + 232
SSL_FALSESTART = 0 + 233
PATH_AS_IS = 0 + 234
PROXY_SERVICE_NAME = 10000 + 235
SERVICE_NAME = 10000 + 236
PIPEWAIT = 0 + 237
DEFAULT_PROTOCOL = 10000 + 238
STREAM_WEIGHT = 0 + 239
STREAM_DEPENDS = 10000 + 240
STREAM_DEPENDS_E = 10000 + 241
TFTP_NO_OPTIONS = 0 + 242
CONNECT_TO = 10000 + 243
TCP_FASTOPEN = 0 + 244
KEEP_SENDING_ON_ERROR = 0 + 245
PROXY_CAINFO = 10000 + 246
PROXY_CAPATH = 10000 + 247
PROXY_SSL_VERIFYPEER = 0 + 248
PROXY_SSL_VERIFYHOST = 0 + 249
PROXY_SSLVERSION = 0 + 250
PROXY_TLSAUTH_USERNAME = 10000 + 251
PROXY_TLSAUTH_PASSWORD = 10000 + 252
PROXY_TLSAUTH_TYPE = 10000 + 253
PROXY_SSLCERT = 10000 + 254
PROXY_SSLCERTTYPE = 10000 + 255
PROXY_SSLKEY = 10000 + 256
PROXY_SSLKEYTYPE = 10000 + 257
PROXY_KEYPASSWD = 10000 + 258
PROXY_SSL_CIPHER_LIST = 10000 + 259
PROXY_CRLFILE = 10000 + 260
PROXY_SSL_OPTIONS = 0 + 261
PRE_PROXY = 10000 + 262
PROXY_PINNEDPUBLICKEY = 10000 + 263
ABSTRACT_UNIX_SOCKET = 10000 + 264
SUPPRESS_CONNECT_HEADERS = 0 + 265
REQUEST_TARGET = 10000 + 266
SOCKS5_AUTH = 0 + 267
SSH_COMPRESSION = 0 + 268
MIMEPOST = 10000 + 269
TIMEVALUE_LARGE = 30000 + 270
HAPPY_EYEBALLS_TIMEOUT_MS = 0 + 271
RESOLVER_START_FUNCTION = 20000 + 272
RESOLVER_START_DATA = 10000 + 273
HAPROXYPROTOCOL = 0 + 274
DNS_SHUFFLE_ADDRESSES = 0 + 275
TLS13_CIPHERS = 10000 + 276
PROXY_TLS13_CIPHERS = 10000 + 277
DISALLOW_USERNAME_IN_URL = 0 + 278
DOH_URL = 10000 + 279
UPLOAD_BUFFERSIZE = 0 + 280
UPKEEP_INTERVAL_MS = 0 + 281
CURLU = 10000 + 282
TRAILERFUNCTION = 20000 + 283
TRAILERDATA = 10000 + 284
HTTP09_ALLOWED = 0 + 285
ALTSVC_CTRL = 0 + 286
ALTSVC = 10000 + 287
MAXAGE_CONN = 0 + 288
SASL_AUTHZID = 10000 + 289
MAIL_RCPT_ALLOWFAILS = 0 + 290
SSLCERT_BLOB = 40000 + 291
SSLKEY_BLOB = 40000 + 292
PROXY_SSLCERT_BLOB = 40000 + 293
PROXY_SSLKEY_BLOB = 40000 + 294
ISSUERCERT_BLOB = 40000 + 295
PROXY_ISSUERCERT = 10000 + 296
PROXY_ISSUERCERT_BLOB = 40000 + 297
SSL_EC_CURVES = 10000 + 298
HSTS_CTRL = 0 + 299
HSTS = 10000 + 300
HSTSREADFUNCTION = 20000 + 301
HSTSREADDATA = 10000 + 302
HSTSWRITEFUNCTION = 20000 + 303
HSTSWRITEDATA = 10000 + 304
AWS_SIGV4 = 10000 + 305
DOH_SSL_VERIFYPEER = 0 + 306
DOH_SSL_VERIFYHOST = 0 + 307
DOH_SSL_VERIFYSTATUS = 0 + 308
CAINFO_BLOB = 40000 + 309
PROXY_CAINFO_BLOB = 40000 + 310
SSH_HOST_PUBLIC_KEY_SHA256 = 10000 + 311
PREREQFUNCTION = 20000 + 312
PREREQDATA = 10000 + 313
MAXLIFETIME_CONN = 0 + 314
MIME_OPTIONS = 0 + 315
SSH_HOSTKEYFUNCTION = 20000 + 316
SSH_HOSTKEYDATA = 10000 + 317
PROTOCOLS_STR = 10000 + 318
REDIR_PROTOCOLS_STR = 10000 + 319
WS_OPTIONS = 0 + 320
CA_CACHE_TIMEOUT = 0 + 321
QUICK_EXIT = 0 + 322
HAPROXY_CLIENT_IP = 10000 + 323
SERVER_RESPONSE_TIMEOUT_MS = 0 + 324
ECH = 10000 + 325
TCP_KEEPCNT = 0 + 326
UPLOAD_FLAGS = 0 + 327
SSL_SIGNATURE_ALGORITHMS = 10000 + 328
HTTPBASEHEADER = 10000 + 1000
SSL_SIG_HASH_ALGS = 10000 + 1001
SSL_ENABLE_ALPS = 0 + 1002
SSL_CERT_COMPRESSION = 10000 + 1003
SSL_ENABLE_TICKET = 0 + 1004
HTTP2_PSEUDO_HEADERS_ORDER = 10000 + 1005
HTTP2_SETTINGS = 10000 + 1006
SSL_PERMUTE_EXTENSIONS = 0 + 1007
HTTP2_WINDOW_UPDATE = 0 + 1008
HTTP2_STREAMS = 10000 + 1010
TLS_GREASE = 0 + 1011
TLS_EXTENSION_ORDER = 10000 + 1012
STREAM_EXCLUSIVE = 0 + 1013
TLS_KEY_USAGE_NO_CHECK = 0 + 1014
TLS_SIGNED_CERT_TIMESTAMPS = 0 + 1015
TLS_STATUS_REQUEST = 0 + 1016
TLS_DELEGATED_CREDENTIALS = 10000 + 1017
TLS_RECORD_SIZE_LIMIT = 0 + 1018
TLS_KEY_SHARES_LIMIT = 0 + 1019
TLS_USE_NEW_ALPS_CODEPOINT = 0 + 1020
HTTP2_NO_PRIORITY = 0 + 1021
PROXY_CREDENTIAL_NO_REUSE = 0 + 1022
if locals().get("WRITEDATA"):
FILE = locals().get("WRITEDATA")
if locals().get("READDATA"):
INFILE = locals().get("READDATA")
if locals().get("HEADERDATA"):
WRITEHEADER = locals().get("HEADERDATA")
class CurlInfo(IntEnum):
"""``CURLINFO_`` constancs extracted from libcurl,
see: https://curl.se/libcurl/c/curl_easy_getinfo.html"""
TEXT = 0
EFFECTIVE_URL = 0x100000 + 1
RESPONSE_CODE = 0x200000 + 2
TOTAL_TIME = 0x300000 + 3
NAMELOOKUP_TIME = 0x300000 + 4
CONNECT_TIME = 0x300000 + 5
PRETRANSFER_TIME = 0x300000 + 6
SIZE_UPLOAD_T = 0x600000 + 7
SIZE_DOWNLOAD_T = 0x600000 + 8
SPEED_DOWNLOAD_T = 0x600000 + 9
SPEED_UPLOAD_T = 0x600000 + 10
HEADER_SIZE = 0x200000 + 11
REQUEST_SIZE = 0x200000 + 12
SSL_VERIFYRESULT = 0x200000 + 13
FILETIME = 0x200000 + 14
FILETIME_T = 0x600000 + 14
CONTENT_LENGTH_DOWNLOAD_T = 0x600000 + 15
CONTENT_LENGTH_UPLOAD_T = 0x600000 + 16
STARTTRANSFER_TIME = 0x300000 + 17
CONTENT_TYPE = 0x100000 + 18
REDIRECT_TIME = 0x300000 + 19
REDIRECT_COUNT = 0x200000 + 20
PRIVATE = 0x100000 + 21
HTTP_CONNECTCODE = 0x200000 + 22
HTTPAUTH_AVAIL = 0x200000 + 23
PROXYAUTH_AVAIL = 0x200000 + 24
OS_ERRNO = 0x200000 + 25
NUM_CONNECTS = 0x200000 + 26
SSL_ENGINES = 0x400000 + 27
COOKIELIST = 0x400000 + 28
FTP_ENTRY_PATH = 0x100000 + 30
REDIRECT_URL = 0x100000 + 31
PRIMARY_IP = 0x100000 + 32
APPCONNECT_TIME = 0x300000 + 33
CERTINFO = 0x400000 + 34
CONDITION_UNMET = 0x200000 + 35
RTSP_SESSION_ID = 0x100000 + 36
RTSP_CLIENT_CSEQ = 0x200000 + 37
RTSP_SERVER_CSEQ = 0x200000 + 38
RTSP_CSEQ_RECV = 0x200000 + 39
PRIMARY_PORT = 0x200000 + 40
LOCAL_IP = 0x100000 + 41
LOCAL_PORT = 0x200000 + 42
ACTIVESOCKET = 0x500000 + 44
TLS_SSL_PTR = 0x400000 + 45
HTTP_VERSION = 0x200000 + 46
PROXY_SSL_VERIFYRESULT = 0x200000 + 47
SCHEME = 0x100000 + 49
TOTAL_TIME_T = 0x600000 + 50
NAMELOOKUP_TIME_T = 0x600000 + 51
CONNECT_TIME_T = 0x600000 + 52
PRETRANSFER_TIME_T = 0x600000 + 53
STARTTRANSFER_TIME_T = 0x600000 + 54
REDIRECT_TIME_T = 0x600000 + 55
APPCONNECT_TIME_T = 0x600000 + 56
RETRY_AFTER = 0x600000 + 57
EFFECTIVE_METHOD = 0x100000 + 58
PROXY_ERROR = 0x200000 + 59
REFERER = 0x100000 + 60
CAINFO = 0x100000 + 61
CAPATH = 0x100000 + 62
XFER_ID = 0x600000 + 63
CONN_ID = 0x600000 + 64
QUEUE_TIME_T = 0x600000 + 65
USED_PROXY = 0x200000 + 66
POSTTRANSFER_TIME_T = 0x600000 + 67
EARLYDATA_SENT_T = 0x600000 + 68
HTTPAUTH_USED = 0x200000 + 69
PROXYAUTH_USED = 0x200000 + 70
LASTONE = 70
if locals().get("RESPONSE_CODE"):
HTTP_CODE = locals().get("RESPONSE_CODE")
class CurlMOpt(IntEnum):
"""``CURLMOPT_`` constancs extracted from libcurl,
see: https://curl.se/libcurl/c/curl_multi_setopt.html"""
SOCKETFUNCTION = 20000 + 1
SOCKETDATA = 10000 + 2
PIPELINING = 0 + 3
TIMERFUNCTION = 20000 + 4
TIMERDATA = 10000 + 5
MAXCONNECTS = 0 + 6
MAX_HOST_CONNECTIONS = 0 + 7
MAX_PIPELINE_LENGTH = 0 + 8
CONTENT_LENGTH_PENALTY_SIZE = 30000 + 9
CHUNK_LENGTH_PENALTY_SIZE = 30000 + 10
PIPELINING_SITE_BL = 10000 + 11
PIPELINING_SERVER_BL = 10000 + 12
MAX_TOTAL_CONNECTIONS = 0 + 13
PUSHFUNCTION = 20000 + 14
PUSHDATA = 10000 + 15
MAX_CONCURRENT_STREAMS = 0 + 16
class CurlECode(IntEnum):
"""``CURLECODE_`` constancs extracted from libcurl,
see: https://curl.se/libcurl/c/libcurl-errors.html"""
OK = 0
UNSUPPORTED_PROTOCOL = 1
FAILED_INIT = 2
URL_MALFORMAT = 3
NOT_BUILT_IN = 4
COULDNT_RESOLVE_PROXY = 5
COULDNT_RESOLVE_HOST = 6
COULDNT_CONNECT = 7
WEIRD_SERVER_REPLY = 8
REMOTE_ACCESS_DENIED = 9
FTP_ACCEPT_FAILED = 10
FTP_WEIRD_PASS_REPLY = 11
FTP_ACCEPT_TIMEOUT = 12
FTP_WEIRD_PASV_REPLY = 13
FTP_WEIRD_227_FORMAT = 14
FTP_CANT_GET_HOST = 15
HTTP2 = 16
FTP_COULDNT_SET_TYPE = 17
PARTIAL_FILE = 18
FTP_COULDNT_RETR_FILE = 19
OBSOLETE20 = 20
QUOTE_ERROR = 21
HTTP_RETURNED_ERROR = 22
WRITE_ERROR = 23
OBSOLETE24 = 24
UPLOAD_FAILED = 25
READ_ERROR = 26
OUT_OF_MEMORY = 27
OPERATION_TIMEDOUT = 28
OBSOLETE29 = 29
FTP_PORT_FAILED = 30
FTP_COULDNT_USE_REST = 31
OBSOLETE32 = 32
RANGE_ERROR = 33
OBSOLETE34 = 34
SSL_CONNECT_ERROR = 35
BAD_DOWNLOAD_RESUME = 36
FILE_COULDNT_READ_FILE = 37
LDAP_CANNOT_BIND = 38
LDAP_SEARCH_FAILED = 39
OBSOLETE40 = 40
OBSOLETE41 = 41
ABORTED_BY_CALLBACK = 42
BAD_FUNCTION_ARGUMENT = 43
OBSOLETE44 = 44
INTERFACE_FAILED = 45
OBSOLETE46 = 46
TOO_MANY_REDIRECTS = 47
UNKNOWN_OPTION = 48
SETOPT_OPTION_SYNTAX = 49
OBSOLETE50 = 50
OBSOLETE51 = 51
GOT_NOTHING = 52
SSL_ENGINE_NOTFOUND = 53
SSL_ENGINE_SETFAILED = 54
SEND_ERROR = 55
RECV_ERROR = 56
OBSOLETE57 = 57
SSL_CERTPROBLEM = 58
SSL_CIPHER = 59
PEER_FAILED_VERIFICATION = 60
BAD_CONTENT_ENCODING = 61
OBSOLETE62 = 62
FILESIZE_EXCEEDED = 63
USE_SSL_FAILED = 64
SEND_FAIL_REWIND = 65
SSL_ENGINE_INITFAILED = 66
LOGIN_DENIED = 67
TFTP_NOTFOUND = 68
TFTP_PERM = 69
REMOTE_DISK_FULL = 70
TFTP_ILLEGAL = 71
TFTP_UNKNOWNID = 72
REMOTE_FILE_EXISTS = 73
TFTP_NOSUCHUSER = 74
OBSOLETE75 = 75
OBSOLETE76 = 76
SSL_CACERT_BADFILE = 77
REMOTE_FILE_NOT_FOUND = 78
SSH = 79
SSL_SHUTDOWN_FAILED = 80
AGAIN = 81
SSL_CRL_BADFILE = 82
SSL_ISSUER_ERROR = 83
FTP_PRET_FAILED = 84
RTSP_CSEQ_ERROR = 85
RTSP_SESSION_ERROR = 86
FTP_BAD_FILE_LIST = 87
CHUNK_FAILED = 88
NO_CONNECTION_AVAILABLE = 89
SSL_PINNEDPUBKEYNOTMATCH = 90
SSL_INVALIDCERTSTATUS = 91
HTTP2_STREAM = 92
RECURSIVE_API_CALL = 93
AUTH_ERROR = 94
HTTP3 = 95
QUIC_CONNECT_ERROR = 96
PROXY = 97
SSL_CLIENTCERT = 98
UNRECOVERABLE_POLL = 99
TOO_LARGE = 100
ECH_REQUIRED = 101
RESERVED115115 = 102
RESERVED116116 = 103
RESERVED117117 = 104
RESERVED118118 = 105
RESERVED119119 = 106
RESERVED120120 = 107
RESERVED121121 = 108
RESERVED122122 = 109
RESERVED123123 = 110
RESERVED124124 = 111
RESERVED125125 = 112
RESERVED126126 = 113
class CurlHttpVersion(IntEnum):
"""``CURL_HTTP_VERSION`` constants from libcurl, see comments for details."""
NONE = 0
V1_0 = 1 # please use HTTP 1.0 in the request */
V1_1 = 2 # please use HTTP 1.1 in the request */
V2_0 = 3 # please use HTTP 2 in the request */
V2TLS = 4 # use version 2 for HTTPS, version 1.1 for HTTP */
V2_PRIOR_KNOWLEDGE = 5 # please use HTTP 2 without HTTP/1.1 Upgrade */
V3 = 30 # Makes use of explicit HTTP/3 with fallback.
V3ONLY = 31 # No fallback
class CurlWsFlag(IntEnum):
"""``CURL_WS_FLAG`` constants from libcurl, see comments for details."""
TEXT = 1 << 0
BINARY = 1 << 1
CONT = 1 << 2
CLOSE = 1 << 3
PING = 1 << 4
OFFSET = 1 << 5
class CurlSslVersion(IntEnum):
"""``CURL_SSLVERSION`` constants from libcurl, see comments for details."""
DEFAULT = 0
TLSv1 = 1
SSLv2 = 2
SSLv3 = 3
TLSv1_0 = 4
TLSv1_1 = 5
TLSv1_2 = 6
TLSv1_3 = 7
MAX_DEFAULT = 1 << 16
class CurlIpResolve(IntEnum):
"""``CURL_IPRESOLVE`` constants from libcurl, see comments for details."""
WHATEVER = 0 # default, uses addresses to all IP versions that your system allows
V4 = 1 # uses only IPv4 addresses/connections
V6 = 2 # uses only IPv6 addresses/connections
WebServer/AIPython/python/lib/python3.11/site-packages/curl_cffi/curl.py
+612
View File
@@ -0,0 +1,612 @@
from __future__ import annotations
import re
import struct
import sys
import warnings
from http.cookies import SimpleCookie
from pathlib import Path
from typing import TYPE_CHECKING, Any, Literal, Optional, Union, cast
import certifi
from ._wrapper import ffi, lib
from .const import CurlECode, CurlHttpVersion, CurlInfo, CurlOpt, CurlWsFlag
from .utils import CurlCffiWarning
DEFAULT_CACERT = certifi.where()
REASON_PHRASE_RE = re.compile(rb"HTTP/\d\.\d [0-9]{3} (.*)")
STATUS_LINE_RE = re.compile(rb"HTTP/(\d\.\d) ([0-9]{3}) (.*)")
if TYPE_CHECKING:
class CurlWsFrame:
age: int
flags: int
offset: int
bytesleft: int
len: int
class CurlError(Exception):
"""Base exception for curl_cffi package"""
def __init__(self, msg, code: Union[CurlECode, Literal[0]] = 0, *args, **kwargs):
super().__init__(msg, *args, **kwargs)
self.code: Union[CurlECode, Literal[0]] = code
CURLINFO_TEXT = 0
CURLINFO_HEADER_IN = 1
CURLINFO_HEADER_OUT = 2
CURLINFO_DATA_IN = 3
CURLINFO_DATA_OUT = 4
CURLINFO_SSL_DATA_IN = 5
CURLINFO_SSL_DATA_OUT = 6
CURL_WRITEFUNC_PAUSE = 0x10000001
CURL_WRITEFUNC_ERROR = 0xFFFFFFFF
@ffi.def_extern()
def debug_function(curl, type_: int, data, size: int, clientp) -> int:
"""ffi callback for curl debug info"""
callback = ffi.from_handle(clientp)
text = ffi.buffer(data, size)[:]
callback(type_, text)
return 0
def bytes_to_hex(b: bytes, uppercase: bool = False) -> str:
"""
Convert a bytes object to a space-separated hex string, e.g. "0a ff 3c".
If uppercase=True, letters will be A–F instead of a–f.
"""
fmt = "{:02X}" if uppercase else "{:02x}"
return " ".join(fmt.format(byte) for byte in b)
def debug_function_default(type_: int, data: bytes) -> None:
PREFIXES = {
CURLINFO_TEXT: "*",
CURLINFO_HEADER_IN: "<",
CURLINFO_HEADER_OUT: ">",
CURLINFO_DATA_IN: "< DATA",
CURLINFO_DATA_OUT: "> DATA",
CURLINFO_SSL_DATA_IN: "< SSL",
CURLINFO_SSL_DATA_OUT: "> SSL",
}
MAX_SHOW_BYTES = 40
prefix = PREFIXES.get(type_, "*")
# always show ssl data in binary format
if type_ in (CURLINFO_SSL_DATA_IN, CURLINFO_SSL_DATA_OUT):
hex_str = bytes_to_hex(data[:MAX_SHOW_BYTES])
postfix = "" if len(data) <= MAX_SHOW_BYTES else "..."
sys.stderr.write(f"{prefix} [{len(data)} bytes]: {hex_str}{postfix}\n")
else:
try:
text = data.decode("utf-8")
sys.stderr.write(f"{prefix} {text}")
if type_ not in (CURLINFO_TEXT, CURLINFO_HEADER_IN, CURLINFO_HEADER_OUT):
sys.stderr.write("\n")
except UnicodeDecodeError:
# Fallback to hex representation of first MAX_SHOW_BYTES bytes
hex_str = bytes_to_hex(data[:MAX_SHOW_BYTES])
postfix = "" if len(data) <= MAX_SHOW_BYTES else "..."
sys.stderr.write(f"{prefix} [{len(data)} bytes]: {hex_str}{postfix}\n")
@ffi.def_extern()
def buffer_callback(ptr, size, nmemb, userdata):
"""ffi callback for curl write function, directly writes to a buffer"""
# assert size == 1
buffer = ffi.from_handle(userdata)
buffer.write(ffi.buffer(ptr, nmemb)[:])
return nmemb * size
def ensure_int(s):
if not s:
return 0
return int(s)
@ffi.def_extern()
def write_callback(ptr, size, nmemb, userdata):
"""ffi callback for curl write function, calls the callback python function"""
# although similar enough to the function above, kept here for performance reasons
callback = ffi.from_handle(userdata)
wrote = callback(ffi.buffer(ptr, nmemb)[:])
wrote = ensure_int(wrote)
if wrote == CURL_WRITEFUNC_PAUSE or wrote == CURL_WRITEFUNC_ERROR: # noqa: SIM109
return wrote
# should make this an exception in future versions
if wrote != nmemb * size:
warnings.warn("Wrote bytes != received bytes.", CurlCffiWarning, stacklevel=2)
return nmemb * size
# Credits: @alexio777 on https://github.com/lexiforest/curl_cffi/issues/4
def slist_to_list(head) -> list[bytes]:
"""Converts curl slist to a python list."""
result = []
ptr = head
while ptr:
result.append(ffi.string(ptr.data))
ptr = ptr.next
lib.curl_slist_free_all(head)
return result
class Curl:
"""
Wrapper for ``curl_easy_*`` functions of libcurl.
"""
def __init__(self, cacert: str = "", debug: bool = False, handle=None) -> None:
"""
Parameters:
cacert: CA cert path to use, by default, certs from ``certifi`` are used.
debug: whether to show curl debug messages.
handle: a curl handle instance from ``curl_easy_init``.
"""
self._curl = handle if handle else lib.curl_easy_init()
self._headers = ffi.NULL
self._proxy_headers = ffi.NULL
self._resolve = ffi.NULL
self._cacert = cacert or DEFAULT_CACERT
self._is_cert_set = False
self._write_handle: Any = None
self._header_handle: Any = None
self._debug_handle: Any = None
self._body_handle: Any = None
# TODO: use CURL_ERROR_SIZE
self._error_buffer = ffi.new("char[]", 256)
self._debug = debug
self._set_error_buffer()
def _set_error_buffer(self) -> None:
ret = lib._curl_easy_setopt(self._curl, CurlOpt.ERRORBUFFER, self._error_buffer)
if ret != 0:
warnings.warn("Failed to set error buffer", CurlCffiWarning, stacklevel=2)
if self._debug:
self.debug()
def debug(self) -> None:
"""Set debug to True"""
self.setopt(CurlOpt.VERBOSE, 1)
self.setopt(CurlOpt.DEBUGFUNCTION, True)
def __del__(self) -> None:
self.close()
def _check_error(self, errcode: int, *args: Any) -> None:
error = self._get_error(errcode, *args)
if error is not None:
raise error
def _get_error(self, errcode: int, *args: Any):
if errcode != 0:
errmsg = ffi.string(self._error_buffer).decode(errors="backslashreplace")
action = " ".join([str(a) for a in args])
return CurlError(
f"Failed to {action}, curl: ({errcode}) {errmsg}. "
"See https://curl.se/libcurl/c/libcurl-errors.html first for more "
"details.",
code=cast(CurlECode, errcode),
)
def setopt(self, option: CurlOpt, value: Any) -> int:
"""Wrapper for ``curl_easy_setopt``.
Args:
option: option to set, using constants from CurlOpt enum
value: value to set, strings will be handled automatically
Returns:
0 if no error, see ``CurlECode``.
"""
input_option = {
# this should be int in curl, but cffi requires pointer for void*
# it will be convert back in the glue c code.
0: "long*",
10000: "char*",
20000: "void*",
30000: "int64_t*", # offset type
40000: "void*", # blob type
}
# print("option", option, "value", value)
# Convert value
value_type = input_option.get((option // 10000) * 10000)
if value_type == "long*" or value_type == "int64_t*":
c_value = ffi.new(value_type, value)
elif option == CurlOpt.WRITEDATA:
c_value = ffi.new_handle(value)
self._write_handle = c_value
lib._curl_easy_setopt(
self._curl, CurlOpt.WRITEFUNCTION, lib.buffer_callback
)
elif option == CurlOpt.HEADERDATA:
c_value = ffi.new_handle(value)
self._header_handle = c_value
lib._curl_easy_setopt(
self._curl, CurlOpt.HEADERFUNCTION, lib.buffer_callback
)
elif option == CurlOpt.WRITEFUNCTION:
c_value = ffi.new_handle(value)
self._write_handle = c_value
lib._curl_easy_setopt(self._curl, CurlOpt.WRITEFUNCTION, lib.write_callback)
option = CurlOpt.WRITEDATA
elif option == CurlOpt.HEADERFUNCTION:
c_value = ffi.new_handle(value)
self._header_handle = c_value
lib._curl_easy_setopt(
self._curl, CurlOpt.HEADERFUNCTION, lib.write_callback
)
option = CurlOpt.HEADERDATA
elif option == CurlOpt.DEBUGFUNCTION:
if value is True:
value = debug_function_default
c_value = ffi.new_handle(value)
self._debug_handle = c_value
lib._curl_easy_setopt(self._curl, CurlOpt.DEBUGFUNCTION, lib.debug_function)
option = CurlOpt.DEBUGDATA
elif value_type == "char*":
c_value = value.encode() if isinstance(value, str) else value
# Must keep a reference, otherwise may be GCed.
if option == CurlOpt.POSTFIELDS:
self._body_handle = c_value
else:
raise NotImplementedError(f"Option unsupported: {option}")
if option == CurlOpt.HTTPHEADER:
for header in value:
self._headers = lib.curl_slist_append(self._headers, header)
ret = lib._curl_easy_setopt(self._curl, option, self._headers)
elif option == CurlOpt.PROXYHEADER:
for proxy_header in value:
self._proxy_headers = lib.curl_slist_append(
self._proxy_headers, proxy_header
)
ret = lib._curl_easy_setopt(self._curl, option, self._proxy_headers)
elif option == CurlOpt.RESOLVE:
for resolve in value:
if isinstance(resolve, str):
resolve = resolve.encode()
self._resolve = lib.curl_slist_append(self._resolve, resolve)
ret = lib._curl_easy_setopt(self._curl, option, self._resolve)
else:
ret = lib._curl_easy_setopt(self._curl, option, c_value)
self._check_error(ret, "setopt", option, value)
if option == CurlOpt.CAINFO:
self._is_cert_set = True
return ret
def getinfo(self, option: CurlInfo) -> Union[bytes, int, float, list]:
"""Wrapper for ``curl_easy_getinfo``. Gets information in response after
curl.perform.
Parameters:
option: option to get info of, using constants from ``CurlInfo`` enum
Returns:
value retrieved from last perform.
"""
ret_option = {
0x100000: "char**",
0x200000: "long*",
0x300000: "double*",
0x400000: "struct curl_slist **",
0x500000: "long*",
0x600000: "int64_t*",
}
ret_cast_option = {
0x100000: ffi.string,
0x200000: int,
0x300000: float,
0x500000: int,
0x600000: int,
}
c_value = ffi.new(ret_option[option & 0xF00000])
ret = lib.curl_easy_getinfo(self._curl, option, c_value)
self._check_error(ret, "getinfo", option)
# cookielist and ssl_engines starts with 0x400000, see also: const.py
if option & 0xF00000 == 0x400000:
return slist_to_list(c_value[0])
if c_value[0] == ffi.NULL:
return b""
return ret_cast_option[option & 0xF00000](c_value[0])
def version(self) -> bytes:
"""Get the underlying libcurl version."""
return ffi.string(lib.curl_version())
def impersonate(self, target: str, default_headers: bool = True) -> int:
"""Set the browser type to impersonate.
Parameters:
target: browser to impersonate.
default_headers: whether to add default headers, like User-Agent.
Returns:
0 if no error.
"""
return lib.curl_easy_impersonate(
self._curl, target.encode(), int(default_headers)
)
def _ensure_cacert(self) -> None:
if not self._is_cert_set:
ret = self.setopt(CurlOpt.CAINFO, self._cacert)
self._check_error(ret, "set cacert")
ret = self.setopt(CurlOpt.PROXY_CAINFO, self._cacert)
self._check_error(ret, "set proxy cacert")
def perform(self, clear_headers: bool = True) -> None:
"""Wrapper for ``curl_easy_perform``, performs a curl request.
Parameters:
clear_headers: clear header slist used in this perform
Raises:
CurlError: if the perform was not successful.
"""
# make sure we set a cacert store
self._ensure_cacert()
# here we go
ret = lib.curl_easy_perform(self._curl)
try:
self._check_error(ret, "perform")
finally:
# cleaning
self.clean_after_perform(clear_headers)
def upkeep(self) -> int:
return lib.curl_easy_upkeep(self._curl)
def clean_after_perform(self, clear_headers: bool = True) -> None:
"""Clean up handles and buffers after ``perform``, called at the end of
``perform``."""
self._write_handle = None
self._header_handle = None
self._debug_handle = None
self._body_handle = None
if clear_headers:
if self._headers != ffi.NULL:
lib.curl_slist_free_all(self._headers)
self._headers = ffi.NULL
if self._proxy_headers != ffi.NULL:
lib.curl_slist_free_all(self._proxy_headers)
self._proxy_headers = ffi.NULL
def duphandle(self) -> Curl:
"""Wrapper for ``curl_easy_duphandle``.
This is not a full copy of entire curl object in python. For example, headers
handle is not copied, you have to set them again."""
new_handle = lib.curl_easy_duphandle(self._curl)
c = Curl(cacert=self._cacert, debug=self._debug, handle=new_handle)
return c
def reset(self) -> None:
"""Reset all curl options, wrapper for ``curl_easy_reset``."""
self._is_cert_set = False
if self._curl is not None:
lib.curl_easy_reset(self._curl)
self._set_error_buffer()
self._resolve = ffi.NULL
def parse_cookie_headers(self, headers: list[bytes]) -> SimpleCookie:
"""Extract ``cookies.SimpleCookie`` from header lines.
Parameters:
headers: list of headers in bytes.
Returns:
A parsed cookies.SimpleCookie instance.
"""
cookie: SimpleCookie = SimpleCookie()
for header in headers:
if header.lower().startswith(b"set-cookie: "):
cookie.load(header[12:].decode()) # len("set-cookie: ") == 12
return cookie
@staticmethod
def get_reason_phrase(status_line: bytes) -> bytes:
"""Extract reason phrase, like ``OK``, ``Not Found`` from response status
line."""
m = REASON_PHRASE_RE.match(status_line)
return m.group(1) if m else b""
@staticmethod
def parse_status_line(status_line: bytes) -> tuple[CurlHttpVersion, int, bytes]:
"""Parse status line.
Returns:
http_version, status_code, and reason phrase
"""
m = STATUS_LINE_RE.match(status_line)
if not m:
return CurlHttpVersion.V1_0, 0, b""
if m.group(1) == "2.0":
http_version = CurlHttpVersion.V2_0
elif m.group(1) == "1.1":
http_version = CurlHttpVersion.V1_1
elif m.group(1) == "1.0":
http_version = CurlHttpVersion.V1_0
else:
http_version = CurlHttpVersion.NONE
status_code = int(m.group(2))
reason = m.group(3)
return http_version, status_code, reason
def close(self) -> None:
"""Close and cleanup curl handle, wrapper for ``curl_easy_cleanup``."""
if self._curl:
lib.curl_easy_cleanup(self._curl)
self._curl = None
ffi.release(self._error_buffer)
self._resolve = ffi.NULL
def ws_recv(self, n: int = 1024) -> tuple[bytes, CurlWsFrame]:
"""Receive a frame from a websocket connection.
Args:
n: maximum data to receive.
Returns:
a tuple of frame content and curl frame meta struct.
Raises:
CurlError: if failed.
"""
buffer = ffi.new("char[]", n)
n_recv = ffi.new("size_t *")
p_frame = ffi.new("struct curl_ws_frame **")
ret = lib.curl_ws_recv(self._curl, buffer, n, n_recv, p_frame)
self._check_error(ret, "WS_RECV")
# Frame meta explained: https://curl.se/libcurl/c/curl_ws_meta.html
frame = p_frame[0]
return ffi.buffer(buffer)[: n_recv[0]], frame
def ws_send(self, payload: bytes, flags: CurlWsFlag = CurlWsFlag.BINARY) -> int:
"""Send data to a websocket connection.
Args:
payload: content to send.
flags: websocket flag to set for the frame, default: binary.
Returns:
0 if no error.
Raises:
CurlError: if failed.
"""
n_sent = ffi.new("size_t *")
buffer = ffi.from_buffer(payload)
ret = lib.curl_ws_send(self._curl, buffer, len(payload), n_sent, 0, flags)
self._check_error(ret, "WS_SEND")
return n_sent[0]
def ws_close(self, code: int = 1000, message: bytes = b"") -> int:
"""Close a websocket connection. Shorthand for :meth:`ws_send`
with close code and message. Note that to completely close the connection,
you must close the curl handle after this call with :meth:`close`.
Args:
code: close code.
message: close message.
Returns:
0 if no error.
Raises:
CurlError: if failed.
"""
return self.ws_send(struct.pack("!H", code) + message)
class CurlMime:
"""Wrapper for the ``curl_mime_`` API."""
def __init__(self, curl: Optional[Curl] = None):
"""
Args:
curl: Curl instance to use.
"""
self._curl = curl if curl else Curl()
self._form = lib.curl_mime_init(self._curl._curl)
def addpart(
self,
name: str,
*,
content_type: Optional[str] = None,
filename: Optional[str] = None,
local_path: Optional[Union[str, bytes, Path]] = None,
data: Optional[bytes] = None,
) -> None:
"""Add a mime part for a mutlipart html form.
Note: You can only use either local_path or data, not both.
Args:
name: name of the field.
content_type: content_type for the field. for example: ``image/png``.
filename: filename for the server.
local_path: file to upload on local disk.
data: file content to upload.
"""
part = lib.curl_mime_addpart(self._form)
ret = lib.curl_mime_name(part, name.encode())
if ret != 0:
raise CurlError("Add field failed.")
# mime type
if content_type is not None:
ret = lib.curl_mime_type(part, content_type.encode())
if ret != 0:
raise CurlError("Add field failed.")
# remote file name
if filename is not None:
ret = lib.curl_mime_filename(part, filename.encode())
if ret != 0:
raise CurlError("Add field failed.")
if local_path and data:
raise CurlError("Can not use local_path and data at the same time.")
# this is a filename
if local_path is not None:
if isinstance(local_path, Path):
local_path_str = str(local_path)
elif isinstance(local_path, bytes):
local_path_str = local_path.decode()
else:
local_path_str = local_path
if not Path(local_path_str).exists():
raise FileNotFoundError(f"File not found at {local_path_str}")
ret = lib.curl_mime_filedata(part, local_path_str.encode())
if ret != 0:
raise CurlError("Add field failed.")
if data is not None:
if not isinstance(data, bytes):
data = str(data).encode()
ret = lib.curl_mime_data(part, data, len(data))
@classmethod
def from_list(cls, files: list[dict]):
"""Create a multipart instance from a list of dict, for keys, see ``addpart``"""
form = cls()
for file in files:
form.addpart(**file)
return form
def attach(self, curl: Optional[Curl] = None) -> None:
"""Attach the mime instance to a curl instance."""
c = curl if curl else self._curl
c.setopt(CurlOpt.MIMEPOST, self._form)
def close(self) -> None:
"""Close the mime instance and underlying files. This method must be called
after ``perform`` or ``request``."""
lib.curl_mime_free(self._form)
self._form = ffi.NULL
def __del__(self) -> None:
self.close()
WebServer/AIPython/python/lib/python3.11/site-packages/curl_cffi/py.typed
+1
View File
@@ -0,0 +1 @@
# Marker file for PEP 561.
WebServer/AIPython/python/lib/python3.11/site-packages/curl_cffi/requests/__init__.py
+171
View File
@@ -0,0 +1,171 @@
__all__ = [
"Session",
"AsyncSession",
"BrowserType",
"BrowserTypeLiteral",
"CurlWsFlag",
"request",
"head",
"get",
"post",
"put",
"patch",
"delete",
"options",
"RequestsError",
"Cookies",
"Headers",
"Request",
"Response",
"AsyncWebSocket",
"WebSocket",
"WebSocketError",
"WebSocketClosed",
"WebSocketTimeout",
"WsCloseCode",
"ExtraFingerprints",
"CookieTypes",
"HeaderTypes",
"ProxySpec",
]
from typing import Optional, TYPE_CHECKING, TypedDict
from ..const import CurlWsFlag
from .cookies import Cookies, CookieTypes
from .errors import RequestsError
from .headers import Headers, HeaderTypes
from .impersonate import BrowserType, BrowserTypeLiteral, ExtraFingerprints
from .models import Request, Response
from .session import (
AsyncSession,
HttpMethod,
ProxySpec,
Session,
ThreadType,
RequestParams,
Unpack,
)
from .websockets import (
AsyncWebSocket,
WebSocket,
WebSocketClosed,
WebSocketError,
WebSocketTimeout,
WsCloseCode,
)
if TYPE_CHECKING:
class SessionRequestParams(RequestParams, total=False):
thread: Optional[ThreadType]
curl_options: Optional[dict]
debug: Optional[bool]
else:
SessionRequestParams = TypedDict
def request(
method: HttpMethod,
url: str,
thread: Optional[ThreadType] = None,
curl_options: Optional[dict] = None,
debug: Optional[bool] = None,
**kwargs: Unpack[RequestParams],
) -> Response:
"""Send an http request.
Parameters:
method: http method for the request: GET/POST/PUT/DELETE etc.
url: url for the requests.
params: query string for the requests.
data: form values(dict/list/tuple) or binary data to use in body,
``Content-Type: application/x-www-form-urlencoded`` will be added if a dict
is given.
json: json values to use in body, `Content-Type: application/json` will be added
automatically.
headers: headers to send.
cookies: cookies to use.
files: not supported, use ``multipart`` instead.
auth: HTTP basic auth, a tuple of (username, password), only basic auth is
supported.
timeout: how many seconds to wait before giving up.
allow_redirects: whether to allow redirection.
max_redirects: max redirect counts, default 30, use -1 for unlimited.
proxies: dict of proxies to use, prefer to use ``proxy`` if they are the same.
format: ``{"http": proxy_url, "https": proxy_url}``.
proxy: proxy to use, format: "http://user@pass:proxy_url".
Can't be used with `proxies` parameter.
proxy_auth: HTTP basic auth for proxy, a tuple of (username, password).
verify: whether to verify https certs.
referer: shortcut for setting referer header.
accept_encoding: shortcut for setting accept-encoding header.
content_callback: a callback function to receive response body.
``def callback(chunk: bytes) -> None:``
impersonate: which browser version to impersonate.
ja3: ja3 string to impersonate.
akamai: akamai string to impersonate.
extra_fp: extra fingerprints options, in complement to ja3 and akamai strings.
thread: thread engine to use for working with other thread implementations.
choices: eventlet, gevent.
default_headers: whether to set default browser headers when impersonating.
default_encoding: encoding for decoding response content if charset is not found
in headers. Defaults to "utf-8". Can be set to a callable for automatic
detection.
quote: Set characters to be quoted, i.e. percent-encoded. Default safe string
is ``!#$%&'()*+,/:;=?@[]~``. If set to a sting, the character will be
removed from the safe string, thus quoted. If set to False, the url will be
kept as is, without any automatic percent-encoding, you must encode the URL
yourself.
curl_options: extra curl options to use.
http_version: limiting http version, defaults to http2.
debug: print extra curl debug info.
interface: which interface to use.
cert: a tuple of (cert, key) filenames for client cert.
stream: streaming the response, default False.
max_recv_speed: maximum receive speed, bytes per second.
multipart: upload files using the multipart format, see examples for details.
discard_cookies: discard cookies from server. Default to False.
Returns:
A ``Response`` object.
"""
debug = False if debug is None else debug
with Session(thread=thread, curl_options=curl_options, debug=debug) as s:
return s.request(method=method, url=url, **kwargs)
def head(url: str, **kwargs: Unpack[SessionRequestParams]):
return request(method="HEAD", url=url, **kwargs)
def get(url: str, **kwargs: Unpack[SessionRequestParams]):
return request(method="GET", url=url, **kwargs)
def post(url: str, **kwargs: Unpack[SessionRequestParams]):
return request(method="POST", url=url, **kwargs)
def put(url: str, **kwargs: Unpack[SessionRequestParams]):
return request(method="PUT", url=url, **kwargs)
def patch(url: str, **kwargs: Unpack[SessionRequestParams]):
return request(method="PATCH", url=url, **kwargs)
def delete(url: str, **kwargs: Unpack[SessionRequestParams]):
return request(method="DELETE", url=url, **kwargs)
def options(url: str, **kwargs: Unpack[SessionRequestParams]):
return request(method="OPTIONS", url=url, **kwargs)
def trace(url: str, **kwargs: Unpack[SessionRequestParams]):
return request(method="TRACE", url=url, **kwargs)
def query(url: str, **kwargs: Unpack[SessionRequestParams]):
return request(method="QUERY", url=url, **kwargs)
WebServer/AIPython/python/lib/python3.11/site-packages/curl_cffi/requests/cookies.py
+364
View File
@@ -0,0 +1,364 @@
# Adapted from: https://github.com/encode/httpx/blob/master/httpx/_models.py,
# which is licensed under the BSD License.
# See https://github.com/encode/httpx/blob/master/LICENSE.md
__all__ = ["Cookies"]
import re
import time
import warnings
from dataclasses import dataclass
from http.cookiejar import Cookie, CookieJar
from http.cookies import _unquote
from typing import Optional, Union
from collections.abc import Iterator, MutableMapping
from urllib.parse import urlparse
from ..utils import CurlCffiWarning
from .errors import CookieConflict, RequestsError
CookieTypes = Union["Cookies", CookieJar, dict[str, str], list[tuple[str, str]]]
@dataclass
class CurlMorsel:
name: str
value: str
hostname: str = ""
subdomains: bool = False
path: str = "/"
secure: bool = False
expires: int = 0
http_only: bool = False
@staticmethod
def parse_bool(s):
return s == "TRUE"
@staticmethod
def dump_bool(s):
return "TRUE" if s else "FALSE"
@classmethod
def from_curl_format(cls, set_cookie_line: bytes):
(
hostname,
subdomains,
path,
secure,
expires,
name,
value,
) = set_cookie_line.decode().split("\t")
if hostname and hostname[0] == "#":
http_only = True
# e.g. #HttpOnly_postman-echo.com
domain = hostname[10:] # len("#HttpOnly_") == 10
else:
http_only = False
domain = hostname
return cls(
hostname=domain,
subdomains=cls.parse_bool(subdomains),
path=path,
secure=cls.parse_bool(secure),
expires=int(expires),
name=name,
value=_unquote(value),
http_only=http_only,
)
def to_curl_format(self):
if not self.hostname:
raise RequestsError(f"Domain not found for cookie {self.name}={self.value}")
return "\t".join(
[
self.hostname,
self.dump_bool(self.subdomains),
self.path,
self.dump_bool(self.secure),
str(self.expires),
self.name,
self.value,
]
)
@classmethod
def from_cookiejar_cookie(cls, cookie: Cookie):
return cls(
name=cookie.name,
value=cookie.value or "",
hostname=cookie.domain,
subdomains=cookie.domain_specified,
path=cookie.path,
secure=cookie.secure,
expires=int(cookie.expires or 0),
http_only=False,
)
def to_cookiejar_cookie(self) -> Cookie:
# the leading dot actually does not mean anything nowadays
# https://stackoverflow.com/a/20884869/1061155
# https://github.com/python/cpython/blob/d6555abfa7384b5a40435a11bdd2aa6bbf8f5cfc/Lib/http/cookiejar.py#L1535
return Cookie(
version=0,
name=self.name,
value=self.value,
port=None,
port_specified=False,
domain=self.hostname,
domain_specified=self.subdomains,
domain_initial_dot=bool(self.hostname.startswith(".")),
path=self.path,
path_specified=bool(self.path),
secure=self.secure,
# using if explicitly to make it clear.
expires=None if self.expires == 0 else self.expires,
discard=self.expires == 0,
comment=None,
comment_url=None,
rest=dict(http_only=f"{self.http_only}"),
rfc2109=False,
)
cut_port_re = re.compile(r":\d+$", re.ASCII)
IPV4_RE = re.compile(r"\.\d+$", re.ASCII)
class Cookies(MutableMapping[str, str]):
"""
HTTP Cookies, as a mutable mapping.
"""
def __init__(self, cookies: Optional[CookieTypes] = None) -> None:
if cookies is None or isinstance(cookies, dict):
self.jar = CookieJar()
if isinstance(cookies, dict):
for key, value in cookies.items():
self.set(key, value)
elif isinstance(cookies, list):
self.jar = CookieJar()
for key, value in cookies:
self.set(key, value)
elif isinstance(cookies, Cookies):
self.jar = CookieJar()
for cookie in cookies.jar:
self.jar.set_cookie(cookie)
else:
self.jar = cookies
def _eff_request_host(self, request) -> str:
"""
Almost equivalent to the eff_request_host function in:
https://github.com/python/cpython/blob/3.11/Lib/http/cookiejar.py#L636
"""
host = urlparse(request.url)[1]
if host == "":
host = request.headers.get("Host", "")
# remove port, if present
host = cut_port_re.sub("", host, 1)
host = host.lower()
if host.find(".") == -1 and not IPV4_RE.search(host):
host += ".local"
return host
def get_cookies_for_curl(self, request) -> list[CurlMorsel]:
"""the process is similar to ``cookiejar.add_cookie_header``, but load all
cookies"""
self.jar._cookies_lock.acquire() # type: ignore
morsels = []
try:
self.jar._policy._now = self._now = int(time.time()) # type: ignore
for cookie in self.jar:
morsel = CurlMorsel.from_cookiejar_cookie(cookie)
if not morsel.hostname:
morsel.hostname = self._eff_request_host(request)
morsels.append(morsel)
finally:
self.jar._cookies_lock.release() # type: ignore
self.jar.clear_expired_cookies()
return morsels
def update_cookies_from_curl(self, morsels: list[CurlMorsel]):
for morsel in morsels:
cookie = morsel.to_cookiejar_cookie()
self.jar.set_cookie(cookie)
self.jar.clear_expired_cookies()
def set(
self, name: str, value: str, domain: str = "", path: str = "/", secure=False
) -> None:
"""
Set a cookie value by name. May optionally include domain and path.
"""
# https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Set-Cookie
if name.startswith("__Secure-") and secure is False:
warnings.warn(
"`secure` changed to True for `__Secure-` prefixed cookies",
CurlCffiWarning,
stacklevel=2,
)
secure = True
elif name.startswith("__Host-") and (secure is False or domain or path != "/"):
warnings.warn(
"`host` changed to True, `domain` removed, `path` changed to `/` "
"for `__Host-` prefixed cookies",
CurlCffiWarning,
stacklevel=2,
)
secure = True
domain = ""
path = "/"
kwargs = {
"version": 0,
"name": name,
"value": value,
"port": None,
"port_specified": False,
"domain": domain,
"domain_specified": bool(domain),
"domain_initial_dot": domain.startswith("."),
"path": path,
"path_specified": bool(path),
"secure": secure,
"expires": None,
"discard": True,
"comment": None,
"comment_url": None,
"rest": {"HttpOnly": None},
"rfc2109": False,
}
cookie = Cookie(**kwargs)
self.jar.set_cookie(cookie)
def get( # type: ignore
self,
name: str,
default: Optional[str] = None,
domain: Optional[str] = None,
path: Optional[str] = None,
) -> Optional[str]:
"""
Get a cookie by name. May optionally include domain and path
in order to specify exactly which cookie to retrieve.
"""
value = None
matched_domain = ""
for cookie in self.jar:
if (
cookie.name == name
and (domain is None or cookie.domain == domain)
and (path is None or cookie.path == path)
):
# if cookies on two different domains do not share a same value
if (
value is not None
and not matched_domain.endswith(cookie.domain)
and not str(cookie.domain).endswith(matched_domain)
and value != cookie.value
):
message = (
f"Multiple cookies exist with name={name} on "
f"{matched_domain} and {cookie.domain}, add domain "
"parameter to suppress this error."
)
raise CookieConflict(message)
value = cookie.value
matched_domain = cookie.domain or ""
if value is None:
return default
return value
def get_dict(
self, domain: Optional[str] = None, path: Optional[str] = None
) -> dict:
"""
Cookies with the same name on different domains may overwrite each other,
do NOT use this function as a method of serialization.
"""
ret = {}
for cookie in self.jar:
if (domain is None or cookie.domain == domain) and (
path is None or cookie.path == path
):
ret[cookie.name] = cookie.value
return ret
def delete(
self,
name: str,
domain: Optional[str] = None,
path: Optional[str] = None,
) -> None:
"""
Delete a cookie by name. May optionally include domain and path
in order to specify exactly which cookie to delete.
"""
if domain is not None and path is not None:
return self.jar.clear(domain, path, name)
remove = [
cookie
for cookie in self.jar
if cookie.name == name
and (domain is None or cookie.domain == domain)
and (path is None or cookie.path == path)
]
for cookie in remove:
self.jar.clear(cookie.domain, cookie.path, cookie.name)
def clear(self, domain: Optional[str] = None, path: Optional[str] = None) -> None:
"""
Delete all cookies. Optionally include a domain and path in
order to only delete a subset of all the cookies.
"""
args = []
if domain is not None:
args.append(domain)
if path is not None:
assert domain is not None
args.append(path)
self.jar.clear(*args)
def update(self, cookies: Optional[CookieTypes] = None) -> None: # type: ignore
cookies = Cookies(cookies)
for cookie in cookies.jar:
self.jar.set_cookie(cookie)
def __setitem__(self, name: str, value: str) -> None:
return self.set(name, value)
def __getitem__(self, name: str) -> str:
value = self.get(name)
if value is None:
raise KeyError(name)
return value
def __delitem__(self, name: str) -> None:
return self.delete(name)
def __len__(self) -> int:
return len(self.jar)
def __iter__(self) -> Iterator[str]:
return (cookie.name for cookie in self.jar)
def __bool__(self) -> bool:
for _ in self.jar:
return True
return False
def __repr__(self) -> str:
cookies_repr = ", ".join(
[
f"<Cookie {cookie.name}={cookie.value} for {cookie.domain} />"
for cookie in self.jar
]
)
return f"<Cookies[{cookies_repr}]>"
WebServer/AIPython/python/lib/python3.11/site-packages/curl_cffi/requests/errors.py
+7
View File
@@ -0,0 +1,7 @@
# for compatibility with 0.5.x
__all__ = ["CurlError", "RequestsError", "CookieConflict", "SessionClosed"]
from ..curl import CurlError
from .exceptions import CookieConflict, SessionClosed
from .exceptions import RequestException as RequestsError
WebServer/AIPython/python/lib/python3.11/site-packages/curl_cffi/requests/exceptions.py
+227
View File
@@ -0,0 +1,227 @@
# Apache 2.0 License
# Vendored from https://github.com/psf/requests/blob/main/src/requests/exceptions.py
# With our own addtions
import json
from typing import Literal, Union
from ..const import CurlECode
from ..curl import CurlError
# Note IOError is an alias of OSError in Python 3.x
class RequestException(CurlError, OSError):
"""Base exception for curl_cffi.requests package"""
def __init__(
self,
msg,
code: Union[CurlECode, Literal[0]] = 0,
response=None,
*args,
**kwargs,
):
super().__init__(msg, code, *args, **kwargs)
self.response = response
class CookieConflict(RequestException):
"""Same cookie exists for different domains."""
class SessionClosed(RequestException):
"""The session has already been closed."""
class ImpersonateError(RequestException):
"""The impersonate config was wrong or impersonate failed."""
# not used
class InvalidJSONError(RequestException):
"""A JSON error occurred. not used"""
# not used
class JSONDecodeError(InvalidJSONError, json.JSONDecodeError):
"""Couldn't decode the text into json. not used"""
class HTTPError(RequestException):
"""An HTTP error occurred."""
class IncompleteRead(HTTPError):
"""Incomplete read of content"""
class ConnectionError(RequestException):
"""A Connection error occurred."""
class DNSError(ConnectionError):
"""Could not resolve"""
class ProxyError(RequestException):
"""A proxy error occurred."""
class SSLError(ConnectionError):
"""An SSL error occurred."""
class CertificateVerifyError(SSLError):
"""Raised when certificate validated has failed"""
class Timeout(RequestException):
"""The request timed out."""
# not used
class ConnectTimeout(ConnectionError, Timeout):
"""The request timed out while trying to connect to the remote server.
Requests that produced this error are safe to retry.
not used
"""
# not used
class ReadTimeout(Timeout):
"""The server did not send any data in the allotted amount of time. not used"""
# not used
class URLRequired(RequestException):
"""A valid URL is required to make a request. not used"""
class TooManyRedirects(RequestException):
"""Too many redirects."""
# not used
class MissingSchema(RequestException, ValueError):
"""The URL scheme (e.g. http or https) is missing. not used"""
class InvalidSchema(RequestException, ValueError):
"""The URL scheme provided is either invalid or unsupported. not used"""
class InvalidURL(RequestException, ValueError):
"""The URL provided was somehow invalid."""
# not used
class InvalidHeader(RequestException, ValueError):
"""The header value provided was somehow invalid. not used"""
# not used
class InvalidProxyURL(InvalidURL):
"""The proxy URL provided is invalid. not used"""
# not used
class ChunkedEncodingError(RequestException):
"""The server declared chunked encoding but sent an invalid chunk. not used"""
# not used
class ContentDecodingError(RequestException):
"""Failed to decode response content. not used"""
# not used
class StreamConsumedError(RequestException, TypeError):
"""The content for this response was already consumed. not used"""
# does not support
class RetryError(RequestException):
"""Custom retries logic failed. not used"""
# not used
class UnrewindableBodyError(RequestException):
"""Requests encountered an error when trying to rewind a body. not used"""
class InterfaceError(RequestException):
"""A specified outgoing interface could not be used."""
# Warnings
# TODO: use this warning as a base
class RequestsWarning(Warning):
"""Base warning for Requests. not used"""
# not used
class FileModeWarning(RequestsWarning, DeprecationWarning):
"""A file was opened in text mode, but Requests determined its binary length.
not used"""
# not used
class RequestsDependencyWarning(RequestsWarning):
"""An imported dependency doesn't match the expected version range."""
CODE2ERROR = {
0: RequestException,
CurlECode.UNSUPPORTED_PROTOCOL: InvalidSchema,
CurlECode.URL_MALFORMAT: InvalidURL,
CurlECode.COULDNT_RESOLVE_PROXY: ProxyError,
CurlECode.COULDNT_RESOLVE_HOST: DNSError,
CurlECode.COULDNT_CONNECT: ConnectionError,
CurlECode.WEIRD_SERVER_REPLY: ConnectionError,
CurlECode.REMOTE_ACCESS_DENIED: ConnectionError,
CurlECode.HTTP2: HTTPError,
CurlECode.HTTP_RETURNED_ERROR: HTTPError,
CurlECode.WRITE_ERROR: RequestException,
CurlECode.READ_ERROR: RequestException,
CurlECode.OUT_OF_MEMORY: RequestException,
CurlECode.OPERATION_TIMEDOUT: Timeout,
CurlECode.SSL_CONNECT_ERROR: SSLError,
CurlECode.INTERFACE_FAILED: InterfaceError,
CurlECode.TOO_MANY_REDIRECTS: TooManyRedirects,
CurlECode.UNKNOWN_OPTION: RequestException,
CurlECode.SETOPT_OPTION_SYNTAX: RequestException,
CurlECode.GOT_NOTHING: ConnectionError,
CurlECode.SSL_ENGINE_NOTFOUND: SSLError,
CurlECode.SSL_ENGINE_SETFAILED: SSLError,
CurlECode.SEND_ERROR: ConnectionError,
CurlECode.RECV_ERROR: ConnectionError,
CurlECode.SSL_CERTPROBLEM: SSLError,
CurlECode.SSL_CIPHER: SSLError,
CurlECode.PEER_FAILED_VERIFICATION: CertificateVerifyError,
CurlECode.BAD_CONTENT_ENCODING: HTTPError,
CurlECode.SSL_ENGINE_INITFAILED: SSLError,
CurlECode.SSL_CACERT_BADFILE: SSLError,
CurlECode.SSL_CRL_BADFILE: SSLError,
CurlECode.SSL_ISSUER_ERROR: SSLError,
CurlECode.SSL_PINNEDPUBKEYNOTMATCH: SSLError,
CurlECode.SSL_INVALIDCERTSTATUS: SSLError,
CurlECode.HTTP2_STREAM: HTTPError,
CurlECode.HTTP3: HTTPError,
CurlECode.QUIC_CONNECT_ERROR: ConnectionError,
CurlECode.PROXY: ProxyError,
CurlECode.SSL_CLIENTCERT: SSLError,
CurlECode.ECH_REQUIRED: SSLError,
CurlECode.PARTIAL_FILE: IncompleteRead,
}
# credits: https://github.com/yt-dlp/yt-dlp/blob/master/yt_dlp/networking/_curlcffi.py#L241
# Unlicense
def code2error(code: Union[CurlECode, Literal[0]], msg: str):
if code == CurlECode.RECV_ERROR and "CONNECT" in msg:
return ProxyError
return CODE2ERROR.get(code, RequestException)
WebServer/AIPython/python/lib/python3.11/site-packages/curl_cffi/requests/headers.py
+347
View File
@@ -0,0 +1,347 @@
# Copied from: https://github.com/encode/httpx/blob/master/httpx/_models.py,
# which is licensed under the BSD License.
# See https://github.com/encode/httpx/blob/master/LICENSE.md
from collections.abc import (
ItemsView,
Iterable,
Iterator,
KeysView,
Mapping,
MutableMapping,
Sequence,
ValuesView,
)
from typing import Any, AnyStr, Optional, Union, cast
HeaderTypes = Union[
"Headers",
Mapping[str, Optional[str]],
Mapping[bytes, Optional[bytes]],
Sequence[tuple[str, str]],
Sequence[tuple[bytes, bytes]],
Sequence[Union[str, bytes]],
]
def to_str(value: Union[str, bytes], encoding: str = "utf-8") -> str:
return value if isinstance(value, str) else value.decode(encoding)
SENSITIVE_HEADERS = {"authorization", "proxy-authorization"}
def obfuscate_sensitive_headers(
items: Iterable[tuple[AnyStr, Optional[AnyStr]]],
) -> Iterator[tuple[AnyStr, Optional[AnyStr]]]:
for k, v in items:
if to_str(k.lower()) in SENSITIVE_HEADERS:
v = b"[secure]" if isinstance(v, bytes) else "[secure]" # type: ignore
yield k, v
def normalize_header_key(
value: Union[str, bytes],
lower: bool,
encoding: Optional[str] = None,
) -> bytes:
"""
Coerce str/bytes into a strictly byte-wise HTTP header key.
"""
bytes_value = (
value if isinstance(value, bytes) else value.encode(encoding or "ascii")
)
return bytes_value.lower() if lower else bytes_value
def normalize_header_value(
value: Union[str, bytes, int, None], encoding: Optional[str] = None
) -> Union[bytes, None]:
"""
Coerce str/bytes into a strictly byte-wise HTTP header value.
"""
if value is None:
return None
if isinstance(value, bytes):
return value
# The default encoding for header value should be latin-1
# See: RFC and https://github.com/python/cpython/blob/bc264eac3ad14dab748e33b3d714c2674872791f/Lib/http/client.py#L1309
if isinstance(value, int):
return str(value).encode()
return cast(str, value).encode(encoding or "latin-1")
class Headers(MutableMapping[str, Optional[str]]):
"""
HTTP headers, as a case-insensitive multi-dict.
"""
def __init__(
self, headers: Optional[HeaderTypes] = None, encoding: Optional[str] = None
):
self._list: list[tuple[bytes, bytes, Optional[bytes]]]
if isinstance(headers, Headers):
self._list = list(headers._list)
encoding = encoding or headers.encoding
elif not headers:
self._list = []
elif isinstance(headers, Mapping):
self._list = [
(
normalize_header_key(k, lower=False, encoding=encoding),
normalize_header_key(k, lower=True, encoding=encoding),
normalize_header_value(v, encoding),
)
for k, v in headers.items()
]
elif isinstance(headers, list):
# list of "Name: Value" pairs
if isinstance(headers[0], (str, bytes)):
sep = ":" if isinstance(headers[0], str) else b":"
h = []
for line in headers:
k, v = line.split(sep, maxsplit=1) # pyright: ignore
h.append((k, v.strip()))
# list of (Name, Value) pairs
elif isinstance(headers[0], tuple):
h = headers
self._list = [
(
normalize_header_key(k, lower=False, encoding=encoding),
normalize_header_key(k, lower=True, encoding=encoding),
normalize_header_value(v, encoding),
)
for k, v in h # pyright: ignore
]
self._encoding = encoding
@property
def encoding(self) -> str:
"""
Header encoding is mandated as ascii, but we allow fallbacks to utf-8
or iso-8859-1.
"""
if self._encoding is None:
for encoding in ["ascii", "utf-8"]:
for key, value in self.raw:
try:
key.decode(encoding)
value.decode(encoding) if value is not None else value
except UnicodeDecodeError:
break
else:
# The else block runs if 'break' did not occur, meaning
# all values fitted the encoding.
self._encoding = encoding
break
else:
# The ISO-8859-1 encoding covers all 256 code points in a byte,
# so will never raise decode errors.
self._encoding = "iso-8859-1"
return self._encoding
@encoding.setter
def encoding(self, value: str) -> None:
self._encoding = value
@property
def raw(self) -> list[tuple[bytes, Optional[bytes]]]:
"""
Returns a list of the raw header items, as byte pairs.
"""
return [(raw_key, value) for raw_key, _, value in self._list]
def keys(self) -> KeysView[str]:
return {key.decode(self.encoding): None for _, key, _ in self._list}.keys()
def values(self) -> ValuesView[Optional[str]]:
values_dict: dict[str, str] = {}
for _, key, value in self._list:
str_key = key.decode(self.encoding)
str_value = value.decode(self.encoding) if value is not None else "None"
if str_key in values_dict:
values_dict[str_key] += f", {str_value}"
else:
values_dict[str_key] = str_value
return values_dict.values()
def items(self) -> ItemsView[str, Optional[str]]:
"""
Return `(key, value)` items of headers. Concatenate headers
into a single comma separated value when a key occurs multiple times.
"""
values_dict: dict[str, str] = {}
for _, key, value in self._list:
str_key = key.decode(self.encoding)
str_value = value.decode(self.encoding) if value is not None else "None"
if str_key in values_dict:
values_dict[str_key] += f", {str_value}"
else:
values_dict[str_key] = str_value
return values_dict.items()
def multi_items(self) -> list[tuple[str, Optional[str]]]:
"""
Return a list of `(key, value)` pairs of headers. Allow multiple
occurrences of the same key without concatenating into a single
comma separated value.
"""
return [
(
key.decode(self.encoding),
value.decode(self.encoding) if value is not None else value,
)
for key, _, value in self._list
]
def get(self, key: str, default: Any = None) -> Any:
"""
Return a header value. If multiple occurrences of the header occur
then concatenate them together with commas.
"""
try:
return self[key]
except KeyError:
return default
def get_list(self, key: str, split_commas: bool = False) -> list[Optional[str]]:
"""
Return a list of all header values for a given key.
If `split_commas=True` is passed, then any comma separated header
values are split into multiple return strings.
"""
get_header_key = key.lower().encode(self.encoding)
values = [
item_value.decode(self.encoding) if item_value is not None else item_value
for _, item_key, item_value in self._list
if item_key.lower() == get_header_key
]
if not split_commas:
return values
split_values = []
for value in values:
split_values.extend([item.strip() for item in value.split(",")]) # type: ignore
return split_values
def update(self, headers: Optional[HeaderTypes] = None) -> None: # type: ignore
headers = Headers(headers)
for key in headers:
if key in self:
self.pop(key)
self._list.extend(headers._list)
def copy(self) -> "Headers":
return Headers(self, encoding=self.encoding)
def __getitem__(self, key: str) -> Optional[str]:
"""
Return a single header value.
If there are multiple headers with the same key, then we concatenate
them with commas. See: https://tools.ietf.org/html/rfc7230#section-3.2.2
"""
normalized_key = key.lower().encode(self.encoding)
items = [
header_value.decode(self.encoding)
if header_value is not None
else header_value
for _, header_key, header_value in self._list
if header_key == normalized_key
]
if items == [None]:
return None
if items:
return ", ".join([str(item) for item in items])
raise KeyError(key)
def __setitem__(self, key: str, value: Optional[str]) -> None:
"""
Set the header `key` to `value`, removing any duplicate entries.
Retains insertion order.
"""
set_key = key.encode(self._encoding or "utf-8")
set_value = (
value.encode(self._encoding or "utf-8") if value is not None else value
)
lookup_key = set_key.lower()
found_indexes = [
idx
for idx, (_, item_key, _) in enumerate(self._list)
if item_key == lookup_key
]
for idx in reversed(found_indexes[1:]):
del self._list[idx]
if found_indexes:
idx = found_indexes[0]
self._list[idx] = (set_key, lookup_key, set_value)
else:
self._list.append((set_key, lookup_key, set_value))
def __delitem__(self, key: str) -> None:
"""
Remove the header `key`.
"""
del_key = key.lower().encode(self.encoding)
pop_indexes = [
idx
for idx, (_, item_key, _) in enumerate(self._list)
if item_key.lower() == del_key
]
if not pop_indexes:
raise KeyError(key)
for idx in reversed(pop_indexes):
del self._list[idx]
def __contains__(self, key: Any) -> bool:
header_key = key.lower().encode(self.encoding)
return header_key in [key for _, key, _ in self._list]
def __iter__(self) -> Iterator[Any]:
return iter(self.keys())
def __len__(self) -> int:
return len(self._list)
def __eq__(self, other: Any) -> bool:
try:
other_headers = Headers(other)
except ValueError:
return False
self_list = [(key, value) for _, key, value in self._list]
other_list = [(key, value) for _, key, value in other_headers._list]
return sorted(self_list) == sorted(other_list)
def __repr__(self) -> str:
class_name = self.__class__.__name__
encoding_str = ""
if self.encoding != "ascii":
encoding_str = f", encoding={self.encoding!r}"
as_list = list(obfuscate_sensitive_headers(self.multi_items()))
as_dict = dict(as_list)
no_duplicate_keys = len(as_dict) == len(as_list)
if no_duplicate_keys:
return f"{class_name}({as_dict!r}{encoding_str})"
return f"{class_name}({as_list!r}{encoding_str})"
WebServer/AIPython/python/lib/python3.11/site-packages/curl_cffi/requests/impersonate.py
+435
View File
@@ -0,0 +1,435 @@
import warnings
from dataclasses import dataclass
from enum import Enum
from typing import Literal, Optional, TypedDict
from ..const import CurlOpt, CurlSslVersion
from ..utils import CurlCffiWarning
BrowserTypeLiteral = Literal[
# Edge
"edge99",
"edge101",
# Chrome
"chrome99",
"chrome100",
"chrome101",
"chrome104",
"chrome107",
"chrome110",
"chrome116",
"chrome119",
"chrome120",
"chrome123",
"chrome124",
"chrome131",
"chrome133a",
"chrome136",
"chrome99_android",
"chrome131_android",
# Safari
"safari153",
"safari155",
"safari170",
"safari172_ios",
"safari180",
"safari180_ios",
"safari184",
"safari184_ios",
"safari260",
"safari260_ios",
# Firefox
"firefox133",
"firefox135",
"tor145",
# alias
"chrome",
"edge",
"safari",
"safari_ios",
"safari_beta",
"safari_ios_beta",
"chrome_android",
"firefox",
# deprecated aliases
"safari15_3",
"safari15_5",
"safari17_0",
"safari17_2_ios",
"safari18_0",
"safari18_0_ios",
"safari18_4",
"safari18_4_ios",
# Canonical names
# "edge_99",
# "edge_101",
# "safari_15.3_macos",
# "safari_15.5_macos",
# "safari_17.2_ios",
# "safari_17.0_macos",
# "safari_18.0_ios",
# "safari_18.0_macos",
]
DEFAULT_CHROME = "chrome136"
DEFAULT_EDGE = "edge101"
DEFAULT_SAFARI = "safari184"
DEFAULT_SAFARI_IOS = "safari184_ios"
DEFAULT_SAFARI_BETA = "safari260"
DEFAULT_SAFARI_IOS_BETA = "safari260_ios"
DEFAULT_CHROME_ANDROID = "chrome131_android"
DEFAULT_FIREFOX = "firefox135"
DEFAULT_TOR = "tor145"
REAL_TARGET_MAP = {
"chrome": "chrome136",
"edge": "edge101",
"safari": "safari184",
"safari_ios": "safari184_ios",
"safari_beta": "safari260",
"safari_ios_beta": "safari260_ios",
"chrome_android": "chrome131_android",
"firefox": "firefox135",
"tor": "tor145",
}
def normalize_browser_type(item):
if item == "chrome": # noqa: SIM116
return DEFAULT_CHROME
elif item == "edge":
return DEFAULT_EDGE
elif item == "safari":
return DEFAULT_SAFARI
elif item == "safari_ios":
return DEFAULT_SAFARI_IOS
elif item == "safari_beta":
return DEFAULT_SAFARI_BETA
elif item == "safari_ios_beta":
return DEFAULT_SAFARI_IOS_BETA
elif item == "chrome_android":
return DEFAULT_CHROME_ANDROID
elif item == "firefox":
return DEFAULT_FIREFOX
elif item == "tor":
return DEFAULT_TOR
else:
return item
class BrowserType(str, Enum): # TODO: remove in version 1.x
edge99 = "edge99"
edge101 = "edge101"
chrome99 = "chrome99"
chrome100 = "chrome100"
chrome101 = "chrome101"
chrome104 = "chrome104"
chrome107 = "chrome107"
chrome110 = "chrome110"
chrome116 = "chrome116"
chrome119 = "chrome119"
chrome120 = "chrome120"
chrome123 = "chrome123"
chrome124 = "chrome124"
chrome131 = "chrome131"
chrome133a = "chrome133a"
chrome136 = "chrome136"
chrome99_android = "chrome99_android"
chrome131_android = "chrome131_android"
safari153 = "safari153"
safari155 = "safari155"
safari170 = "safari170"
safari172_ios = "safari172_ios"
safari180 = "safari180"
safari180_ios = "safari180_ios"
safari184 = "safari184"
safari184_ios = "safari184_ios"
safari260 = "safari260"
safari260_ios = "safari260_ios"
firefox133 = "firefox133"
firefox135 = "firefox135"
tor145 = "tor145"
# deprecated aliases
safari15_3 = "safari15_3"
safari15_5 = "safari15_5"
safari17_0 = "safari17_0"
safari17_2_ios = "safari17_2_ios"
safari18_0 = "safari18_0"
safari18_0_ios = "safari18_0_ios"
@dataclass
class ExtraFingerprints:
tls_min_version: int = CurlSslVersion.TLSv1_2
tls_grease: bool = False
tls_permute_extensions: bool = False
tls_cert_compression: Literal["zlib", "brotli"] = "brotli"
tls_signature_algorithms: Optional[list[str]] = None
tls_delegated_credential: str = ""
tls_record_size_limit: int = 0
http2_stream_weight: int = 256
http2_stream_exclusive: int = 1
http2_no_priority: bool = False
class ExtraFpDict(TypedDict, total=False):
tls_min_version: int
tls_grease: bool
tls_permute_extensions: bool
tls_cert_compression: Literal["zlib", "brotli"]
tls_signature_algorithms: Optional[list[str]]
tls_delegated_credential: str
tls_record_size_limit: int
http2_stream_weight: int
http2_stream_exclusive: int
http2_no_priority: bool
# TLS version are in the format of 0xAABB, where AA is major version and BB is minor
# version. As of today, the major version is always 03.
TLS_VERSION_MAP = {
0x0301: CurlSslVersion.TLSv1_0, # 769
0x0302: CurlSslVersion.TLSv1_1, # 770
0x0303: CurlSslVersion.TLSv1_2, # 771
0x0304: CurlSslVersion.TLSv1_3, # 772
}
# A list of the possible cipher suite ids. Taken from
# http://www.iana.org/assignments/tls-parameters/tls-parameters.xml
# via BoringSSL
TLS_CIPHER_NAME_MAP = {
0x000A: "TLS_RSA_WITH_3DES_EDE_CBC_SHA",
0x002F: "TLS_RSA_WITH_AES_128_CBC_SHA",
0x0033: "TLS_DHE_RSA_WITH_AES_128_CBC_SHA",
0x0035: "TLS_RSA_WITH_AES_256_CBC_SHA",
0x0039: "TLS_DHE_RSA_WITH_AES_256_CBC_SHA",
0x003C: "TLS_RSA_WITH_AES_128_CBC_SHA256",
0x003D: "TLS_RSA_WITH_AES_256_CBC_SHA256",
0x0067: "TLS_DHE_RSA_WITH_AES_128_CBC_SHA256",
0x006B: "TLS_DHE_RSA_WITH_AES_256_CBC_SHA256",
0x008C: "TLS_PSK_WITH_AES_128_CBC_SHA",
0x008D: "TLS_PSK_WITH_AES_256_CBC_SHA",
0x009C: "TLS_RSA_WITH_AES_128_GCM_SHA256",
0x009D: "TLS_RSA_WITH_AES_256_GCM_SHA384",
0x009E: "TLS_DHE_RSA_WITH_AES_128_GCM_SHA256",
0x009F: "TLS_DHE_RSA_WITH_AES_256_GCM_SHA384",
0x1301: "TLS_AES_128_GCM_SHA256",
0x1302: "TLS_AES_256_GCM_SHA384",
0x1303: "TLS_CHACHA20_POLY1305_SHA256",
0xC008: "TLS_ECDHE_ECDSA_WITH_3DES_EDE_CBC_SHA",
0xC009: "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA",
0xC00A: "TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA",
0xC012: "TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA",
0xC013: "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA",
0xC014: "TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA",
0xC023: "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256",
0xC024: "TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384",
0xC027: "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256",
0xC028: "TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384",
0xC02B: "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256",
0xC02C: "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384",
0xC02F: "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",
0xC030: "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384",
0xC035: "TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA",
0xC036: "TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA",
0xCCA8: "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256",
0xCCA9: "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256",
0xCCAC: "TLS_ECDHE_PSK_WITH_CHACHA20_POLY1305_SHA256",
}
# RFC tls extensions: https://datatracker.ietf.org/doc/html/rfc6066
# IANA list: https://www.iana.org/assignments/tls-extensiontype-values/tls-extensiontype-values.xhtml
TLS_EXTENSION_NAME_MAP = {
0: "server_name",
1: "max_fragment_length",
2: "client_certificate_url",
3: "trusted_ca_keys",
4: "truncated_hmac",
5: "status_request",
6: "user_mapping",
7: "client_authz",
8: "server_authz",
9: "cert_type",
10: "supported_groups", # (renamed from "elliptic_curves")
11: "ec_point_formats",
12: "srp",
13: "signature_algorithms",
14: "use_srtp",
15: "heartbeat",
16: "application_layer_protocol_negotiation",
17: "status_request_v2",
18: "signed_certificate_timestamp",
19: "client_certificate_type",
20: "server_certificate_type",
21: "padding",
22: "encrypt_then_mac",
23: "extended_master_secret",
24: "token_binding",
25: "cached_info",
26: "tls_lts",
27: "compress_certificate",
28: "record_size_limit",
29: "pwd_protect",
30: "pwd_clear",
31: "password_salt",
32: "ticket_pinning",
33: "tls_cert_with_extern_psk",
34: "delegated_credential",
35: "session_ticket", # (renamed from "SessionTicket TLS")
36: "TLMSP",
37: "TLMSP_proxying",
38: "TLMSP_delegate",
39: "supported_ekt_ciphers",
# 40:"Reserved",
41: "pre_shared_key",
42: "early_data",
43: "supported_versions",
44: "cookie",
45: "psk_key_exchange_modes",
# 46:"Reserved",
47: "certificate_authorities",
48: "oid_filters",
49: "post_handshake_auth",
50: "signature_algorithms_cert",
51: "key_share",
52: "transparency_info",
# 53:"connection_id", # (deprecated)
54: "connection_id",
55: "external_id_hash",
56: "external_session_id",
57: "quic_transport_parameters",
58: "ticket_request",
59: "dnssec_chain",
60: "sequence_number_encryption_algorithms",
61: "rrc",
17513: "application_settings", # BoringSSL private usage
17613: "application_settings new", # BoringSSL private usage
# 62-2569:"Unassigned
# 2570:"Reserved
# 2571-6681:"Unassigned
# 6682:"Reserved
# 6683-10793:"Unassigned
# 10794:"Reserved
# 10795-14905:"Unassigned
# 14906:"Reserved
# 14907-19017:"Unassigned
# 19018:"Reserved
# 19019-23129:"Unassigned
# 23130:"Reserved
# 23131-27241:"Unassigned
# 27242:"Reserved
# 27243-31353:"Unassigned
# 31354:"Reserved
# 31355-35465:"Unassigned
# 35466:"Reserved
# 35467-39577:"Unassigned
# 39578:"Reserved
# 39579-43689:"Unassigned
# 43690:"Reserved
# 43691-47801:"Unassigned
# 47802:"Reserved
# 47803-51913:"Unassigned
# 51914:"Reserved
# 51915-56025:"Unassigned
# 56026:"Reserved
# 56027-60137:"Unassigned
# 60138:"Reserved
# 60139-64249:"Unassigned
# 64250:"Reserved
# 64251-64767:"Unassigned
64768: "ech_outer_extensions",
# 64769-65036:"Unassigned
65037: "encrypted_client_hello",
# 65038-65279:"Unassigned
# 65280:"Reserved for Private Use
65281: "renegotiation_info",
# 65282-65535:"Reserved for Private Use
}
TLS_EC_CURVES_MAP = {
19: "P-192",
21: "P-224",
23: "P-256",
24: "P-384",
25: "P-521",
29: "X25519",
256: "ffdhe2048",
257: "ffdhe3072",
4588: "X25519MLKEM768",
25497: "X25519Kyber768Draft00",
}
def toggle_extension(curl, extension_id: int, enable: bool):
# ECH
if extension_id == 65037:
if enable:
curl.setopt(CurlOpt.ECH, "grease")
else:
curl.setopt(CurlOpt.ECH, "")
# compress certificate
elif extension_id == 27:
if enable:
warnings.warn(
"Cert compression setting to brotli, "
"you had better specify which to use: zlib/brotli",
CurlCffiWarning,
stacklevel=1,
)
curl.setopt(CurlOpt.SSL_CERT_COMPRESSION, "brotli")
else:
curl.setopt(CurlOpt.SSL_CERT_COMPRESSION, "")
# ALPS: application settings
elif extension_id == 17513:
if enable:
curl.setopt(CurlOpt.SSL_ENABLE_ALPS, 1)
else:
curl.setopt(CurlOpt.SSL_ENABLE_ALPS, 0)
elif extension_id == 17613:
if enable:
curl.setopt(CurlOpt.SSL_ENABLE_ALPS, 1)
curl.setopt(CurlOpt.TLS_USE_NEW_ALPS_CODEPOINT, 1)
else:
curl.setopt(CurlOpt.SSL_ENABLE_ALPS, 0)
curl.setopt(CurlOpt.TLS_USE_NEW_ALPS_CODEPOINT, 0)
# server_name
elif extension_id == 0:
raise NotImplementedError(
"It's unlikely that the server_name(0) extension being changed."
)
# ALPN
elif extension_id == 16:
if enable:
curl.setopt(CurlOpt.SSL_ENABLE_ALPN, 1)
else:
curl.setopt(CurlOpt.SSL_ENABLE_ALPN, 0)
# status_request
elif extension_id == 5:
if enable:
curl.setopt(CurlOpt.TLS_STATUS_REQUEST, 1)
# signed_certificate_timestamps
elif extension_id == 18:
if enable:
curl.setopt(CurlOpt.TLS_SIGNED_CERT_TIMESTAMPS, 1)
# session_ticket
elif extension_id == 35:
if enable:
curl.setopt(CurlOpt.SSL_ENABLE_TICKET, 1)
else:
curl.setopt(CurlOpt.SSL_ENABLE_TICKET, 0)
# padding, should be ignored
elif extension_id == 21:
pass # type: ignore
# firefox extension, toggled by extra_fp
elif extension_id in [34, 28]:
pass
else:
raise NotImplementedError(
f"This extension({extension_id}) can not be toggled for now, it may be "
"updated later."
)
WebServer/AIPython/python/lib/python3.11/site-packages/curl_cffi/requests/models.py
+314
View File
@@ -0,0 +1,314 @@
from contextlib import suppress
import queue
import re
import warnings
from concurrent.futures import Future
from typing import Any, Callable, Optional, Union
from collections.abc import Awaitable
from ..curl import Curl
from ..utils import CurlCffiWarning
from .cookies import Cookies
from .exceptions import HTTPError, RequestException
from .headers import Headers
# Use orjson if present
try:
from orjson import loads
except ImportError:
from json import loads
with suppress(ImportError):
from markdownify import markdownify as md
import readability as rd
CHARSET_RE = re.compile(r"charset=([\w-]+)")
STREAM_END = object()
def clear_queue(q: queue.Queue):
with q.mutex:
q.queue.clear()
q.all_tasks_done.notify_all()
q.unfinished_tasks = 0
class Request:
"""Representing a sent request."""
def __init__(self, url: str, headers: Headers, method: str):
self.url = url
self.headers = headers
self.method = method
class Response:
"""Contains information the server sends.
Attributes:
url: url used in the request.
content: response body in bytes.
text: response body in str.
status_code: http status code.
reason: http response reason, such as OK, Not Found.
ok: is status_code in [200, 400)?
headers: response headers.
cookies: response cookies.
elapsed: how many seconds the request cost.
encoding: http body encoding.
charset: alias for encoding.
primary_ip: primary ip of the server.
primary_port: primary port of the server.
local_ip: local ip used in this connection.
local_port: local port used in this connection.
charset_encoding: encoding specified by the Content-Type header.
default_encoding: encoding for decoding response content if charset is not found
in headers. Defaults to "utf-8". Can be set to a callable for automatic
detection.
redirect_count: how many redirects happened.
redirect_url: the final redirected url.
http_version: http version used.
history: history redirections, only headers are available.
"""
def __init__(self, curl: Optional[Curl] = None, request: Optional[Request] = None):
self.curl = curl
self.request = request
self.url = ""
self.content = b""
self.status_code = 200
self.reason = "OK"
self.ok = True
self.headers = Headers()
self.cookies = Cookies()
self.elapsed = 0.0
self.default_encoding: Union[str, Callable[[bytes], str]] = "utf-8"
self.redirect_count = 0
self.redirect_url = ""
self.http_version = 0
self.primary_ip: str = ""
self.primary_port: int = 0
self.local_ip: str = ""
self.local_port: int = 0
self.history: list[dict[str, Any]] = []
self.infos: dict[str, Any] = {}
self.queue: Optional[queue.Queue] = None
self.stream_task: Optional[Future] = None
self.astream_task: Optional[Awaitable] = None
self.quit_now = None
@property
def charset(self) -> str:
"""Alias for encoding."""
return self.encoding
@property
def encoding(self) -> str:
"""
Determines the encoding to decode byte content into text.
The method follows a specific priority to decide the encoding:
1. If ``.encoding`` has been explicitly set, it is used.
2. The encoding specified by the ``charset`` parameter in the ``Content-Type``
header.
3. The encoding specified by the ``default_encoding`` attribute. This can either
be a string (e.g., "utf-8") or a callable for charset autodetection.
"""
if not hasattr(self, "_encoding"):
encoding = self.charset_encoding
if encoding is None:
if isinstance(self.default_encoding, str):
encoding = self.default_encoding
elif callable(self.default_encoding):
encoding = self.default_encoding(self.content)
self._encoding = encoding or "utf-8"
return self._encoding
@encoding.setter
def encoding(self, value: str) -> None:
if hasattr(self, "_text"):
raise ValueError("Cannot set encoding after text has been accessed")
self._encoding = value
@property
def charset_encoding(self) -> Optional[str]:
"""Return the encoding, as specified by the Content-Type header."""
content_type = self.headers.get("Content-Type")
if content_type:
charset_match = CHARSET_RE.search(content_type)
return charset_match.group(1) if charset_match else None
return None
@property
def text(self) -> str:
if not hasattr(self, "_text"):
if not self.content:
self._text = ""
else:
self._text = self._decode(self.content)
return self._text
def markdown(self) -> str:
doc = rd.Document(self.content)
title = doc.title()
summary = doc.summary(html_partial=True)
body_as_md = md(f"<h1>{title}</h1><main>{summary}</main>")
return body_as_md
def _decode(self, content: bytes) -> str:
try:
return content.decode(self.encoding, errors="replace")
except (UnicodeDecodeError, LookupError):
return content.decode("utf-8-sig")
def raise_for_status(self):
"""Raise an error if status code is not in [200, 400)"""
if not self.ok:
raise HTTPError(f"HTTP Error {self.status_code}: {self.reason}", 0, self)
def iter_lines(self, chunk_size=None, decode_unicode=False, delimiter=None):
"""
iterate streaming content line by line, separated by ``\\n``.
Copied from: https://requests.readthedocs.io/en/latest/_modules/requests/models/
which is under the License: Apache 2.0
"""
pending = None
for chunk in self.iter_content(
chunk_size=chunk_size, decode_unicode=decode_unicode
):
if pending is not None:
chunk = pending + chunk
lines = chunk.split(delimiter) if delimiter else chunk.splitlines()
pending = (
lines.pop()
if lines and lines[-1] and chunk and lines[-1][-1] == chunk[-1]
else None
)
yield from lines
if pending is not None:
yield pending
def iter_content(self, chunk_size=None, decode_unicode=False):
"""
iterate streaming content chunk by chunk in bytes.
"""
if chunk_size:
warnings.warn(
"chunk_size is ignored, there is no way to tell curl that.",
CurlCffiWarning,
stacklevel=2,
)
if decode_unicode:
raise NotImplementedError()
assert self.queue and self.curl, "stream mode is not enabled."
while True:
chunk = self.queue.get()
# re-raise the exception if something wrong happened.
if isinstance(chunk, RequestException):
self.curl.reset()
raise chunk
# end of stream.
if chunk is STREAM_END:
break
yield chunk
def json(self, **kw):
"""return a parsed json object of the content."""
return loads(self.content, **kw)
def close(self):
"""Close the streaming connection, only valid in stream mode."""
if self.quit_now:
self.quit_now.set()
if self.stream_task:
self.stream_task.result()
async def aiter_lines(self, chunk_size=None, decode_unicode=False, delimiter=None):
"""
iterate streaming content line by line, separated by ``\\n``.
Copied from: https://requests.readthedocs.io/en/latest/_modules/requests/models/
which is under the License: Apache 2.0
"""
pending = None
async for chunk in self.aiter_content(
chunk_size=chunk_size, decode_unicode=decode_unicode
):
if pending is not None:
chunk = pending + chunk
lines = chunk.split(delimiter) if delimiter else chunk.splitlines()
pending = (
lines.pop()
if lines and lines[-1] and chunk and lines[-1][-1] == chunk[-1]
else None
)
for line in lines:
yield line
if pending is not None:
yield pending
async def aiter_content(self, chunk_size=None, decode_unicode=False):
"""
iterate streaming content chunk by chunk in bytes.
"""
if chunk_size:
warnings.warn(
"chunk_size is ignored, there is no way to tell curl that.",
CurlCffiWarning,
stacklevel=2,
)
if decode_unicode:
raise NotImplementedError()
assert self.queue and self.curl, "stream mode is not enabled."
while True:
chunk = await self.queue.get()
# re-raise the exception if something wrong happened.
if isinstance(chunk, RequestException):
await self.aclose()
raise chunk
# end of stream.
if chunk is STREAM_END:
await self.aclose()
return
yield chunk
async def atext(self) -> str:
"""
Return a decoded string.
"""
return self._decode(await self.acontent())
async def acontent(self) -> bytes:
"""wait and read the streaming content in one bytes object."""
chunks = []
async for chunk in self.aiter_content():
chunks.append(chunk)
return b"".join(chunks)
async def aclose(self):
"""Close the streaming connection, only valid in stream mode."""
if self.astream_task:
await self.astream_task
# It prints the status code of the response instead of the object's memory location.
def __repr__(self) -> str:
return f"<Response [{self.status_code}]>"
WebServer/AIPython/python/lib/python3.11/site-packages/curl_cffi/requests/session.py
+1112
View File
File diff suppressed because it is too large Load Diff
WebServer/AIPython/python/lib/python3.11/site-packages/curl_cffi/requests/utils.py
+698
View File
@@ -0,0 +1,698 @@
from __future__ import annotations
__all__ = ["HttpVersionLiteral", "set_curl_options", "not_set"]
import asyncio
import math
import queue
import warnings
from collections import Counter
from io import BytesIO
from json import dumps
from typing import TYPE_CHECKING, Any, Callable, Final, Literal, Optional, Union, cast
from urllib.parse import ParseResult, parse_qsl, quote, urlencode, urljoin, urlparse
from ..const import CurlHttpVersion, CurlOpt, CurlSslVersion
from ..curl import CURL_WRITEFUNC_ERROR, CurlMime
from ..utils import CurlCffiWarning
from .cookies import Cookies
from .exceptions import ImpersonateError, InvalidURL
from .headers import Headers
from .impersonate import (
TLS_CIPHER_NAME_MAP,
TLS_EC_CURVES_MAP,
TLS_VERSION_MAP,
ExtraFingerprints,
normalize_browser_type,
toggle_extension,
)
from .models import Request
if TYPE_CHECKING:
from ..curl import Curl
from .cookies import CookieTypes
from .headers import HeaderTypes
from .impersonate import BrowserTypeLiteral, ExtraFpDict
from .session import ProxySpec
HttpMethod = Literal[
"GET", "POST", "PUT", "DELETE", "OPTIONS", "HEAD", "TRACE", "PATCH", "QUERY"
]
HttpVersionLiteral = Literal["v1", "v2", "v2tls", "v2_prior_knowledge", "v3", "v3only"]
SAFE_CHARS = set("!#$%&'()*+,/:;=?@[]~")
not_set: Final[Any] = object()
# ruff: noqa: SIM116
def normalize_http_version(
version: Union[CurlHttpVersion, HttpVersionLiteral],
) -> CurlHttpVersion:
if version == "v1":
return CurlHttpVersion.V1_1
elif version == "v3":
return CurlHttpVersion.V3
elif version == "v3only":
return CurlHttpVersion.V3ONLY
elif version == "v2":
return CurlHttpVersion.V2_0
elif version == "v2tls":
return CurlHttpVersion.V2TLS
elif version == "v2_prior_knowledge":
return CurlHttpVersion.V2_PRIOR_KNOWLEDGE
return version # type: ignore
def is_absolute_url(url: str) -> bool:
"""Check if the provided url is an absolute url"""
parsed_url = urlparse(url)
return bool(parsed_url.scheme and parsed_url.hostname)
def quote_path_and_params(url: str, quote_str: str = ""):
safe = "".join(SAFE_CHARS - set(quote_str))
parsed_url = urlparse(url)
parsed_get_args = parse_qsl(parsed_url.query, keep_blank_values=True)
encoded_get_args = urlencode(parsed_get_args, doseq=True, safe=safe)
return ParseResult(
parsed_url.scheme,
parsed_url.netloc,
quote(parsed_url.path, safe=safe),
parsed_url.params,
encoded_get_args,
parsed_url.fragment,
).geturl()
def update_url_params(url: str, params: Union[dict, list, tuple]) -> str:
"""Add URL query params to provided URL being aware of existing.
Args:
url: string of target URL
params: dict containing requested params to be added
Returns:
string with updated URL
>> url = 'http://stackoverflow.com/test?answers=true'
>> new_params = {'answers': False, 'data': ['some','values']}
>> update_url_params(url, new_params)
'http://stackoverflow.com/test?data=some&data=values&answers=false'
"""
# No need to unquote, since requote_uri will be called later.
parsed_url = urlparse(url)
# Extracting URL arguments from parsed URL, NOTE the result is a list, not dict
parsed_get_args = parse_qsl(parsed_url.query, keep_blank_values=True)
# Merging URL arguments dict with new params
old_args_counter = Counter(x[0] for x in parsed_get_args)
if isinstance(params, dict):
params = list(params.items())
new_args_counter = Counter(x[0] for x in params)
for key, value in params:
# Bool and Dict values should be converted to json-friendly values
if isinstance(value, (bool, dict)):
value = dumps(value)
# 1 to 1 mapping, we have to search and update it.
if old_args_counter.get(key) == 1 and new_args_counter.get(key) == 1:
parsed_get_args = [
(x if x[0] != key else (key, value)) for x in parsed_get_args
]
else:
parsed_get_args.append((key, value))
# Converting URL argument to proper query string
encoded_get_args = urlencode(parsed_get_args, doseq=True)
# Creating new parsed result object based on provided with new
# URL arguments. Same thing happens inside of urlparse.
new_url = ParseResult(
parsed_url.scheme,
parsed_url.netloc,
parsed_url.path,
parsed_url.params,
encoded_get_args,
parsed_url.fragment,
).geturl()
return new_url
# Adapted from: https://github.com/psf/requests/blob/1ae6fc3137a11e11565ed22436aa1e77277ac98c/src%2Frequests%2Futils.py#L633-L682
# License: Apache 2.0
# The unreserved URI characters (RFC 3986)
UNRESERVED_SET = frozenset(
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" + "0123456789-._~"
)
def unquote_unreserved(uri: str) -> str:
"""Un-escape any percent-escape sequences in a URI that are unreserved
characters. This leaves all reserved, illegal and non-ASCII bytes encoded.
"""
parts = uri.split("%")
for i in range(1, len(parts)):
h = parts[i][0:2]
if len(h) == 2 and h.isalnum():
try:
c = chr(int(h, 16))
except ValueError as e:
raise InvalidURL(f"Invalid percent-escape sequence: '{h}'") from e
if c in UNRESERVED_SET:
parts[i] = c + parts[i][2:]
else:
parts[i] = f"%{parts[i]}"
else:
parts[i] = f"%{parts[i]}"
return "".join(parts)
def requote_uri(uri: str) -> str:
"""Re-quote the given URI.
This function passes the given URI through an unquote/quote cycle to
ensure that it is fully and consistently quoted.
"""
safe_with_percent = "!#$%&'()*+,/:;=?@[]~|"
safe_without_percent = "!#$&'()*+,/:;=?@[]~|"
try:
# Unquote only the unreserved characters
# Then quote only illegal characters (do not quote reserved,
# unreserved, or '%')
return quote(unquote_unreserved(uri), safe=safe_with_percent)
except InvalidURL:
# We couldn't unquote the given URI, so let's try quoting it, but
# there may be unquoted '%'s in the URI. We need to make sure they're
# properly quoted so they do not cause issues elsewhere.
return quote(uri, safe=safe_without_percent)
# TODO: should we move this function to headers.py?
def update_header_line(
header_lines: list[str], key: str, value: str, replace: bool = False
):
"""Update header line list by key value pair."""
found = False
for idx, line in enumerate(header_lines):
if line.lower().startswith(key.lower() + ":"):
found = True
if replace:
header_lines[idx] = f"{key}: {value}"
break
if not found:
header_lines.append(f"{key}: {value}")
def peek_queue(q: queue.Queue, default=None):
try:
return q.queue[0]
except IndexError:
return default
def peek_aio_queue(q: asyncio.Queue, default=None):
try:
return q._queue[0] # type: ignore
except IndexError:
return default
def toggle_extensions_by_ids(curl: Curl, extension_ids):
# TODO: find a better representation, rather than magic numbers
default_enabled = {0, 10, 11, 13, 16, 23, 35, 43, 45, 51, 65281}
to_enable_ids = extension_ids - default_enabled
for ext_id in to_enable_ids:
toggle_extension(curl, ext_id, enable=True)
# print("to_enable: ", to_enable_ids)
to_disable_ids = default_enabled - extension_ids
for ext_id in to_disable_ids:
toggle_extension(curl, ext_id, enable=False)
# print("to_disable: ", to_disable_ids)
def set_ja3_options(curl: Curl, ja3: str, permute: bool = False):
"""
Detailed explanation: https://engineering.salesforce.com/tls-fingerprinting-with-ja3-and-ja3s-247362855967/
"""
tls_version, ciphers, extensions, curves, curve_formats = ja3.split(",")
curl_tls_version = TLS_VERSION_MAP[int(tls_version)]
curl.setopt(CurlOpt.SSLVERSION, curl_tls_version | CurlSslVersion.MAX_DEFAULT)
assert curl_tls_version == CurlSslVersion.TLSv1_2, "Only TLS v1.2 works for now."
cipher_names = []
for cipher in ciphers.split("-"):
cipher_id = int(cipher)
cipher_name = TLS_CIPHER_NAME_MAP.get(cipher_id)
if not cipher_name:
raise ImpersonateError(f"Cipher {hex(cipher_id)} is not found")
cipher_names.append(cipher_name)
curl.setopt(CurlOpt.SSL_CIPHER_LIST, ":".join(cipher_names))
if extensions.endswith("-21"):
extensions = extensions[:-3]
warnings.warn(
"Padding(21) extension found in ja3 string, whether to add it should "
"be managed by the SSL engine. The TLS client hello packet may contain "
"or not contain this extension, any of which should be correct.",
CurlCffiWarning,
stacklevel=1,
)
extension_ids = set(int(e) for e in extensions.split("-"))
toggle_extensions_by_ids(curl, extension_ids)
if not permute:
curl.setopt(CurlOpt.TLS_EXTENSION_ORDER, extensions)
curve_names = []
for curve in curves.split("-"):
curve_id = int(curve)
curve_name = TLS_EC_CURVES_MAP[curve_id]
curve_names.append(curve_name)
curl.setopt(CurlOpt.SSL_EC_CURVES, ":".join(curve_names))
assert int(curve_formats) == 0, "Only curve_formats == 0 is supported."
def set_akamai_options(curl: Curl, akamai: str):
"""
Detailed explanation: https://www.blackhat.com/docs/eu-17/materials/eu-17-Shuster-Passive-Fingerprinting-Of-HTTP2-Clients-wp.pdf
"""
settings, window_update, streams, header_order = akamai.split("|")
# For compatiblity with tls.peet.ws
settings = settings.replace(",", ";")
curl.setopt(CurlOpt.HTTP_VERSION, CurlHttpVersion.V2_0)
curl.setopt(CurlOpt.HTTP2_SETTINGS, settings)
curl.setopt(CurlOpt.HTTP2_WINDOW_UPDATE, int(window_update))
if streams != "0":
curl.setopt(CurlOpt.HTTP2_STREAMS, streams)
# m,a,s,p -> masp
# curl-impersonate only accepts masp format, without commas.
curl.setopt(CurlOpt.HTTP2_PSEUDO_HEADERS_ORDER, header_order.replace(",", ""))
def set_extra_fp(curl: Curl, fp: ExtraFingerprints):
if fp.tls_signature_algorithms:
curl.setopt(CurlOpt.SSL_SIG_HASH_ALGS, ",".join(fp.tls_signature_algorithms))
curl.setopt(CurlOpt.SSLVERSION, fp.tls_min_version | CurlSslVersion.MAX_DEFAULT)
curl.setopt(CurlOpt.TLS_GREASE, int(fp.tls_grease))
curl.setopt(CurlOpt.SSL_PERMUTE_EXTENSIONS, int(fp.tls_permute_extensions))
curl.setopt(CurlOpt.SSL_CERT_COMPRESSION, fp.tls_cert_compression)
curl.setopt(CurlOpt.STREAM_WEIGHT, fp.http2_stream_weight)
curl.setopt(CurlOpt.STREAM_EXCLUSIVE, fp.http2_stream_exclusive)
if fp.tls_delegated_credential:
curl.setopt(CurlOpt.TLS_DELEGATED_CREDENTIALS, fp.tls_delegated_credential)
if fp.tls_record_size_limit:
curl.setopt(CurlOpt.TLS_RECORD_SIZE_LIMIT, fp.tls_record_size_limit)
if fp.http2_no_priority:
curl.setopt(CurlOpt.HTTP2_NO_PRIORITY, fp.http2_no_priority)
def set_curl_options(
curl: Curl,
method: HttpMethod,
url: str,
*,
params_list: list[Union[dict, list, tuple, None]] = [], # noqa: B006
base_url: Optional[str] = None,
data: Optional[Union[dict[str, str], list[tuple], str, BytesIO, bytes]] = None,
json: Optional[dict | list] = None,
headers_list: list[Optional[HeaderTypes]] = [], # noqa: B006
cookies_list: list[Optional[CookieTypes]] = [], # noqa: B006
files: Optional[dict] = None,
auth: Optional[tuple[str, str]] = None,
timeout: Optional[Union[float, tuple[float, float], object]] = not_set,
allow_redirects: Optional[bool] = True,
max_redirects: Optional[int] = 30,
proxies_list: list[Optional[ProxySpec]] = [], # noqa: B006
proxy: Optional[str] = None,
proxy_auth: Optional[tuple[str, str]] = None,
verify_list: list[Union[bool, str, None]] = [], # noqa: B006
referer: Optional[str] = None,
accept_encoding: Optional[str] = "gzip, deflate, br, zstd",
content_callback: Optional[Callable] = None,
impersonate: Optional[Union[BrowserTypeLiteral, str]] = None,
ja3: Optional[str] = None,
akamai: Optional[str] = None,
extra_fp: Optional[Union[ExtraFingerprints, ExtraFpDict]] = None,
default_headers: bool = True,
quote: Union[str, Literal[False]] = "",
http_version: Optional[Union[CurlHttpVersion, HttpVersionLiteral]] = None,
interface: Optional[str] = None,
cert: Optional[Union[str, tuple[str, str]]] = None,
stream: Optional[bool] = None,
max_recv_speed: int = 0,
multipart: Optional[CurlMime] = None,
queue_class: Any = None,
event_class: Any = None,
curl_options: Optional[dict[CurlOpt, str]] = None,
):
c = curl
method = method.upper() # type: ignore
# method
if method == "POST":
c.setopt(CurlOpt.POST, 1)
elif method != "GET":
c.setopt(CurlOpt.CUSTOMREQUEST, method.encode())
if method == "HEAD":
c.setopt(CurlOpt.NOBODY, 1)
# url
base_params, params = params_list
if base_params:
url = update_url_params(url, base_params)
if params:
url = update_url_params(url, params)
if base_url:
url = urljoin(base_url, url)
if quote:
url = quote_path_and_params(url, quote_str=quote)
if quote is not False:
url = requote_uri(url)
c.setopt(CurlOpt.URL, url.encode())
# data/body/json
if isinstance(data, (dict, list, tuple)):
body = urlencode(data).encode()
elif isinstance(data, str):
body = data.encode()
elif isinstance(data, BytesIO):
body = data.read()
elif isinstance(data, bytes):
body = data
elif data is None:
body = b""
else:
raise TypeError("data must be dict/list/tuple, str, BytesIO or bytes")
if json is not None:
body = dumps(json, separators=(",", ":")).encode()
# Tell libcurl to be aware of bodies and related headers when,
# 1. POST/PUT/PATCH, even if the body is empty, it's up to curl to decide what to do
# 2. GET/DELETE with body, although it's against the RFC, some applications.
# e.g. Elasticsearch, use this.
if body or method in ("POST", "PUT", "PATCH"):
c.setopt(CurlOpt.POSTFIELDS, body)
# necessary if body contains '\0'
c.setopt(CurlOpt.POSTFIELDSIZE, len(body))
if method == "GET":
c.setopt(CurlOpt.CUSTOMREQUEST, method)
# headers
base_headers, headers = headers_list
# let headers encoding take precedence over base headers encoding
encoding = headers.encoding if isinstance(headers, Headers) else None
h = Headers(base_headers, encoding=encoding)
h.update(headers)
# remove Host header if it's unnecessary, otherwise curl may get confused.
# Host header will be automatically added by curl if it's not present.
# https://github.com/lexiforest/curl_cffi/issues/119
host_header = h.get("Host")
if host_header is not None:
u = urlparse(url)
if host_header == u.netloc or host_header == u.hostname:
h.pop("Host", None)
# Make curl always include empty headers.
# See: https://stackoverflow.com/a/32911474/1061155
header_lines = []
for k, v in h.multi_items():
if v is None:
header_lines.append(f"{k}:") # Explictly disable this header
elif v == "":
header_lines.append(f"{k};") # Add an empty valued header
else:
header_lines.append(f"{k}: {v}")
# Add content-type if missing
if json is not None:
update_header_line(header_lines, "Content-Type", "application/json")
if isinstance(data, dict) and method != "POST":
update_header_line(
header_lines, "Content-Type", "application/x-www-form-urlencoded"
)
if isinstance(data, (str, bytes)):
update_header_line(header_lines, "Content-Type", "application/octet-stream")
# Never send `Expect` header.
update_header_line(header_lines, "Expect", "", replace=True)
c.setopt(CurlOpt.HTTPHEADER, [h.encode() for h in header_lines])
req = Request(url, h, method)
# cookies
c.setopt(CurlOpt.COOKIEFILE, b"") # always enable the curl cookie engine first
c.setopt(CurlOpt.COOKIELIST, "ALL") # remove all the old cookies first.
base_cookies, cookies = cookies_list
if base_cookies:
for morsel in base_cookies.get_cookies_for_curl(req): # type: ignore
curl.setopt(CurlOpt.COOKIELIST, morsel.to_curl_format())
if cookies:
temp_cookies = Cookies(cookies)
for morsel in temp_cookies.get_cookies_for_curl(req):
curl.setopt(CurlOpt.COOKIELIST, morsel.to_curl_format())
# files
if files:
raise NotImplementedError(
"files is not supported, use `multipart`. See examples here: "
"https://github.com/lexiforest/curl_cffi/blob/main/examples/upload.py"
)
# multipart
if multipart:
# multipart will overrides postfields
for k, v in cast(dict, data or {}).items():
multipart.addpart(name=k, data=v.encode() if isinstance(v, str) else v)
c.setopt(CurlOpt.MIMEPOST, multipart._form)
# auth
if auth:
username, password = auth
c.setopt(CurlOpt.USERNAME, username.encode()) # pyright: ignore [reportPossiblyUnboundVariable=none]
c.setopt(CurlOpt.PASSWORD, password.encode()) # pyright: ignore [reportPossiblyUnboundVariable=none]
# timeout
if timeout is None:
timeout = 0 # indefinitely
if isinstance(timeout, tuple):
connect_timeout, read_timeout = timeout
all_timeout = connect_timeout + read_timeout
c.setopt(CurlOpt.CONNECTTIMEOUT_MS, int(connect_timeout * 1000))
if not stream:
c.setopt(CurlOpt.TIMEOUT_MS, int(all_timeout * 1000))
else:
# trick from: https://github.com/lexiforest/curl_cffi/issues/156
c.setopt(CurlOpt.LOW_SPEED_LIMIT, 1)
c.setopt(CurlOpt.LOW_SPEED_TIME, math.ceil(all_timeout))
elif isinstance(timeout, (int, float)):
if not stream:
c.setopt(CurlOpt.TIMEOUT_MS, int(timeout * 1000))
else:
c.setopt(CurlOpt.CONNECTTIMEOUT_MS, int(timeout * 1000))
c.setopt(CurlOpt.LOW_SPEED_LIMIT, 1)
c.setopt(CurlOpt.LOW_SPEED_TIME, math.ceil(timeout))
# allow_redirects
c.setopt(CurlOpt.FOLLOWLOCATION, int(allow_redirects)) # type: ignore
# max_redirects
c.setopt(CurlOpt.MAXREDIRS, max_redirects)
# proxies
base_proxies, proxies = proxies_list
if proxy and proxies:
raise TypeError("Cannot specify both 'proxy' and 'proxies'")
if proxy:
proxies = {"all": proxy}
if proxies is None:
proxies = base_proxies
if proxies:
# Turn on proxy_credential_no_reuse, which has the following benefits:
# 1. New connection will be made when proxy username changed
# 2. New TLS session will be created based on proxy address, i.e. when accessing
# the same site with different proxies, TLS session won't leak previous IP.
c.setopt(CurlOpt.PROXY_CREDENTIAL_NO_REUSE, 1)
parts = urlparse(url)
proxy = cast(Optional[str], proxies.get(parts.scheme, proxies.get("all")))
if parts.hostname:
proxy = (
proxies.get( # type: ignore
f"{parts.scheme}://{parts.hostname}",
proxies.get(f"all://{parts.hostname}"),
)
or proxy
)
if proxy is not None:
c.setopt(CurlOpt.PROXY, proxy)
if parts.scheme == "https":
if proxy.startswith("https://"):
warnings.warn(
"Make sure you are using https over https proxy, otherwise, "
"the proxy prefix should be 'http://' not 'https://', "
"see: https://github.com/lexiforest/curl_cffi/issues/6",
CurlCffiWarning,
stacklevel=2,
)
# For https site with http tunnel proxy, tell curl to enable tunneling
if not proxy.startswith("socks"):
c.setopt(CurlOpt.HTTPPROXYTUNNEL, 1)
# proxy_auth
if proxy_auth:
username, password = proxy_auth
c.setopt(CurlOpt.PROXYUSERNAME, username.encode())
c.setopt(CurlOpt.PROXYPASSWORD, password.encode())
# verify
base_verify, verify = verify_list
if verify is False or not base_verify and verify is None:
c.setopt(CurlOpt.SSL_VERIFYPEER, 0)
c.setopt(CurlOpt.SSL_VERIFYHOST, 0)
# cert for this single request
if isinstance(verify, str):
c.setopt(CurlOpt.CAINFO, verify)
# cert for the session
if verify in (None, True) and isinstance(base_verify, str):
c.setopt(CurlOpt.CAINFO, base_verify)
# referer
if referer:
c.setopt(CurlOpt.REFERER, referer.encode())
# accept_encoding
if accept_encoding is not None:
c.setopt(CurlOpt.ACCEPT_ENCODING, accept_encoding.encode())
# cert
if cert:
if isinstance(cert, str):
c.setopt(CurlOpt.SSLCERT, cert)
else:
cert, key = cert
c.setopt(CurlOpt.SSLCERT, cert)
c.setopt(CurlOpt.SSLKEY, key)
# impersonate
if impersonate:
impersonate = normalize_browser_type(impersonate)
ret = c.impersonate(impersonate, default_headers=default_headers) # type: ignore
if ret != 0:
raise ImpersonateError(f"Impersonating {impersonate} is not supported")
# extra_fp options
if extra_fp:
if isinstance(extra_fp, dict):
extra_fp = ExtraFingerprints(**extra_fp)
if impersonate:
warnings.warn(
"Extra fingerprints was altered after impersonated version was set.",
CurlCffiWarning,
stacklevel=1,
)
set_extra_fp(c, extra_fp)
# ja3 string
if ja3:
if impersonate:
warnings.warn(
"JA3 fingerprint was altered after impersonated version was set.",
CurlCffiWarning,
stacklevel=1,
)
permute = False
if isinstance(extra_fp, ExtraFingerprints) and extra_fp.tls_permute_extensions:
permute = True
if isinstance(extra_fp, dict) and extra_fp.get("tls_permute_extensions"):
permute = True
set_ja3_options(c, ja3, permute=permute)
# akamai string
if akamai:
if impersonate:
warnings.warn(
"Akamai fingerprint was altered after impersonated version was set.",
CurlCffiWarning,
stacklevel=1,
)
set_akamai_options(c, akamai)
# http_version, after impersonate, which will change this to http2
if http_version:
http_version = normalize_http_version(http_version)
c.setopt(CurlOpt.HTTP_VERSION, http_version)
buffer = None
q = None
header_recved = None
quit_now = None
if stream:
q = queue_class()
header_recved = event_class()
quit_now = event_class()
def qput(chunk):
if not header_recved.is_set():
header_recved.set()
if quit_now.is_set():
return CURL_WRITEFUNC_ERROR
q.put_nowait(chunk)
return len(chunk)
c.setopt(CurlOpt.WRITEFUNCTION, qput)
elif content_callback is not None:
c.setopt(CurlOpt.WRITEFUNCTION, content_callback)
else:
buffer = BytesIO()
c.setopt(CurlOpt.WRITEDATA, buffer)
header_buffer = BytesIO()
c.setopt(CurlOpt.HEADERDATA, header_buffer)
# interface
if interface:
c.setopt(CurlOpt.INTERFACE, interface.encode())
# max_recv_speed
# do not check, since 0 is a valid value to disable it
c.setopt(CurlOpt.MAX_RECV_SPEED_LARGE, max_recv_speed)
# set extra options, after all others, because it will alter some options
if curl_options:
for option, setting in curl_options.items():
c.setopt(option, setting)
return req, buffer, header_buffer, q, header_recved, quit_now
WebServer/AIPython/python/lib/python3.11/site-packages/curl_cffi/requests/websockets.py
+839
View File
@@ -0,0 +1,839 @@
from __future__ import annotations
import asyncio
import struct
from enum import IntEnum
from functools import partial
from json import dumps, loads
from select import select
from typing import (
TYPE_CHECKING,
Any,
Callable,
Literal,
Optional,
TypeVar,
Union,
)
import warnings
from curl_cffi.utils import CurlCffiWarning
from ..aio import CURL_SOCKET_BAD, get_selector
from ..const import CurlECode, CurlInfo, CurlOpt, CurlWsFlag
from ..curl import Curl, CurlError
from .exceptions import SessionClosed, Timeout
from .utils import not_set, set_curl_options
if TYPE_CHECKING:
from typing_extensions import Self
from ..const import CurlHttpVersion
from ..curl import CurlWsFrame
from .cookies import CookieTypes
from .headers import HeaderTypes
from .impersonate import BrowserTypeLiteral, ExtraFingerprints, ExtraFpDict
from .session import AsyncSession, ProxySpec
T = TypeVar("T")
ON_DATA_T = Callable[["WebSocket", bytes, CurlWsFrame], None]
ON_MESSAGE_T = Callable[["WebSocket", Union[bytes, str]], None]
ON_ERROR_T = Callable[["WebSocket", CurlError], None]
ON_OPEN_T = Callable[["WebSocket"], None]
ON_CLOSE_T = Callable[["WebSocket", int, str], None]
# We need a partial for dumps() because a custom function may not accept the parameter
dumps = partial(dumps, separators=(",", ":"))
class WsCloseCode(IntEnum):
"""See: https://www.iana.org/assignments/websocket/websocket.xhtml"""
OK = 1000
GOING_AWAY = 1001
PROTOCOL_ERROR = 1002
UNSUPPORTED_DATA = 1003
UNKNOWN = 1005
ABNORMAL_CLOSURE = 1006
INVALID_DATA = 1007
POLICY_VIOLATION = 1008
MESSAGE_TOO_BIG = 1009
MANDATORY_EXTENSION = 1010
INTERNAL_ERROR = 1011
SERVICE_RESTART = 1012
TRY_AGAIN_LATER = 1013
BAD_GATEWAY = 1014
TLS_HANDSHAKE = 1015
UNAUTHORIZED = 3000
FORBIDDEN = 3003
TIMEOUT = 3008
class WebSocketError(CurlError):
"""WebSocket-specific error."""
def __init__(
self, message: str, code: Union[WsCloseCode, CurlECode, Literal[0]] = 0
):
super().__init__(message, code) # type: ignore
class WebSocketClosed(WebSocketError, SessionClosed):
"""WebSocket is already closed."""
class WebSocketTimeout(WebSocketError, Timeout):
"""WebSocket operation timed out."""
async def aselect(
fd,
mode: Literal["read", "write"] = "read",
*,
loop: asyncio.AbstractEventLoop,
timeout: Optional[float] = None,
) -> bool:
future = loop.create_future()
if mode == "read":
loop.add_reader(fd, future.set_result, None)
future.add_done_callback(lambda _: loop.remove_reader(fd))
elif mode == "write":
loop.add_writer(fd, future.set_result, None)
future.add_done_callback(lambda _: loop.remove_writer(fd))
else:
raise ValueError(f"Invalid mode: {mode}. Must be 'read' or 'write'")
try:
await asyncio.wait_for(future, timeout)
except asyncio.TimeoutError:
return False
return True
class BaseWebSocket:
def __init__(self, curl: Curl, *, autoclose: bool = True, debug: bool = False):
self._curl: Curl = curl
self.autoclose: bool = autoclose
self._close_code: Optional[int] = None
self._close_reason: Optional[str] = None
self.debug = debug
self.closed = False
@property
def curl(self):
if self._curl is not_set:
self._curl = Curl(debug=self.debug)
return self._curl
@property
def close_code(self) -> Optional[int]:
"""The WebSocket close code, if the connection has been closed."""
return self._close_code
@property
def close_reason(self) -> Optional[str]:
"""The WebSocket close reason, if the connection has been closed."""
return self._close_reason
@staticmethod
def _pack_close_frame(code: int, reason: bytes) -> bytes:
return struct.pack("!H", code) + reason
@staticmethod
def _unpack_close_frame(frame: bytes) -> tuple[int, str]:
if len(frame) < 2:
code = WsCloseCode.UNKNOWN
reason = ""
else:
try:
code = struct.unpack_from("!H", frame)[0]
reason = frame[2:].decode()
except UnicodeDecodeError as e:
raise WebSocketError(
"Invalid close message", WsCloseCode.INVALID_DATA
) from e
except Exception as e:
raise WebSocketError(
"Invalid close frame", WsCloseCode.PROTOCOL_ERROR
) from e
else:
if (
code not in WsCloseCode._value2member_map_
or code == WsCloseCode.UNKNOWN
):
raise WebSocketError(
f"Invalid close code: {code}", WsCloseCode.PROTOCOL_ERROR
)
return code, reason
def terminate(self):
"""Terminate the underlying connection."""
self.closed = True
self.curl.close()
EventTypeLiteral = Literal["open", "close", "data", "message", "error"]
class WebSocket(BaseWebSocket):
"""A WebSocket implementation using libcurl."""
def __init__(
self,
curl: Union[Curl, Any] = not_set,
*,
autoclose: bool = True,
skip_utf8_validation: bool = False,
debug: bool = False,
on_open: Optional[ON_OPEN_T] = None,
on_close: Optional[ON_CLOSE_T] = None,
on_data: Optional[ON_DATA_T] = None,
on_message: Optional[ON_MESSAGE_T] = None,
on_error: Optional[ON_ERROR_T] = None,
):
"""
Args:
autoclose: whether to close the WebSocket after receiving a close frame.
skip_utf8_validation: whether to skip UTF-8 validation for text frames in
run_forever().
debug: print extra curl debug info.
on_open: open callback, ``def on_open(ws)``
on_close: close callback, ``def on_close(ws, code, reason)``
on_data: raw data receive callback, ``def on_data(ws, data, frame)``
on_message: message receive callback, ``def on_message(ws, message)``
on_error: error callback, ``def on_error(ws, exception)``
"""
super().__init__(curl=curl, autoclose=autoclose, debug=debug)
self.skip_utf8_validation = skip_utf8_validation
self._emitters: dict[EventTypeLiteral, Callable] = {}
if on_open:
self._emitters["open"] = on_open
if on_close:
self._emitters["close"] = on_close
if on_data:
self._emitters["data"] = on_data
if on_message:
self._emitters["message"] = on_message
if on_error:
self._emitters["error"] = on_error
def __iter__(self) -> WebSocket:
if self.closed:
raise WebSocketClosed("WebSocket is closed")
return self
def __next__(self) -> bytes:
msg, flags = self.recv()
if flags & CurlWsFlag.CLOSE:
raise StopIteration
return msg
def _emit(self, event_type: EventTypeLiteral, *args) -> None:
callback = self._emitters.get(event_type)
if callback:
try:
callback(self, *args)
except Exception as e:
error_callback = self._emitters.get("error")
if error_callback:
error_callback(self, e)
else:
warnings.warn(
f"WebSocket callback '{event_type}' failed",
CurlCffiWarning,
stacklevel=2,
)
def connect(
self,
url: str,
params: Optional[Union[dict, list, tuple]] = None,
headers: Optional[HeaderTypes] = None,
cookies: Optional[CookieTypes] = None,
auth: Optional[tuple[str, str]] = None,
timeout: Optional[Union[float, tuple[float, float], object]] = not_set,
allow_redirects: bool = True,
max_redirects: int = 30,
proxies: Optional[ProxySpec] = None,
proxy: Optional[str] = None,
proxy_auth: Optional[tuple[str, str]] = None,
verify: Optional[bool] = None,
referer: Optional[str] = None,
accept_encoding: Optional[str] = "gzip, deflate, br",
impersonate: Optional[BrowserTypeLiteral] = None,
ja3: Optional[str] = None,
akamai: Optional[str] = None,
extra_fp: Optional[Union[ExtraFingerprints, ExtraFpDict]] = None,
default_headers: bool = True,
quote: Union[str, Literal[False]] = "",
http_version: Optional[CurlHttpVersion] = None,
interface: Optional[str] = None,
cert: Optional[Union[str, tuple[str, str]]] = None,
max_recv_speed: int = 0,
curl_options: Optional[dict[CurlOpt, str]] = None,
):
"""Connect to the WebSocket.
libcurl automatically handles pings and pongs.
ref: https://curl.se/libcurl/c/libcurl-ws.html
Args:
url: url for the requests.
params: query string for the requests.
headers: headers to send.
cookies: cookies to use.
auth: HTTP basic auth, a tuple of (username, password), only basic auth is
supported.
timeout: how many seconds to wait before giving up.
allow_redirects: whether to allow redirection.
max_redirects: max redirect counts, default 30, use -1 for unlimited.
proxies: dict of proxies to use, prefer to use ``proxy`` if they are the
same. format: ``{"http": proxy_url, "https": proxy_url}``.
proxy: proxy to use, format: "http://user@pass:proxy_url".
Can't be used with `proxies` parameter.
proxy_auth: HTTP basic auth for proxy, a tuple of (username, password).
verify: whether to verify https certs.
referer: shortcut for setting referer header.
accept_encoding: shortcut for setting accept-encoding header.
impersonate: which browser version to impersonate.
ja3: ja3 string to impersonate.
akamai: akamai string to impersonate.
extra_fp: extra fingerprints options, in complement to ja3 and akamai str.
default_headers: whether to set default browser headers.
default_encoding: encoding for decoding response content if charset is not
found in headers. Defaults to "utf-8". Can be set to a callable for
automatic detection.
quote: Set characters to be quoted, i.e. percent-encoded. Default safe
string is ``!#$%&'()*+,/:;=?@[]~``. If set to a sting, the character
will be removed from the safe string, thus quoted. If set to False, the
url will be kept as is, without any automatic percent-encoding, you must
encode the URL yourself.
curl_options: extra curl options to use.
http_version: limiting http version, defaults to http2.
interface: which interface to use.
cert: a tuple of (cert, key) filenames for client cert.
max_recv_speed: maximum receive speed, bytes per second.
curl_options: extra curl options to use.
"""
curl = self.curl
set_curl_options(
curl=curl,
method="GET",
url=url,
params_list=[None, params],
headers_list=[None, headers],
cookies_list=[None, cookies],
auth=auth,
timeout=timeout,
allow_redirects=allow_redirects,
max_redirects=max_redirects,
proxies_list=[None, proxies],
proxy=proxy,
proxy_auth=proxy_auth,
verify_list=[None, verify],
referer=referer,
accept_encoding=accept_encoding,
impersonate=impersonate,
ja3=ja3,
akamai=akamai,
extra_fp=extra_fp,
default_headers=default_headers,
quote=quote,
http_version=http_version,
interface=interface,
max_recv_speed=max_recv_speed,
cert=cert,
curl_options=curl_options,
)
# Magic number defined in: https://curl.se/docs/websocket.html
curl.setopt(CurlOpt.CONNECT_ONLY, 2)
curl.perform()
return self
def recv_fragment(self) -> tuple[bytes, CurlWsFrame]:
"""Receive a single curl websocket fragment as bytes."""
if self.closed:
raise WebSocketClosed("WebSocket is already closed")
chunk, frame = self.curl.ws_recv()
if frame.flags & CurlWsFlag.CLOSE:
try:
self._close_code, self._close_reason = self._unpack_close_frame(chunk)
except WebSocketError as e:
# Follow the spec to close the connection
# Errors do not respect autoclose
self._close_code = e.code
self.close(e.code)
raise
if self.autoclose:
self.close()
return chunk, frame
def recv(self) -> tuple[bytes, int]:
"""
Receive a frame as bytes. libcurl splits frames into fragments, so we have to
collect all the chunks for a frame.
"""
chunks = []
flags = 0
sock_fd = self.curl.getinfo(CurlInfo.ACTIVESOCKET)
if sock_fd == CURL_SOCKET_BAD:
raise WebSocketError(
"Invalid active socket", CurlECode.NO_CONNECTION_AVAILABLE
)
while True:
try:
# Try to receive the first fragment first
chunk, frame = self.recv_fragment()
flags = frame.flags
chunks.append(chunk)
if frame.bytesleft == 0 and flags & CurlWsFlag.CONT == 0:
break
except CurlError as e:
if e.code == CurlECode.AGAIN:
# According to https://curl.se/libcurl/c/curl_ws_recv.html
# > in real application: wait for socket here, e.g. using select()
_, _, _ = select([sock_fd], [], [], 0.5)
else:
raise
return b"".join(chunks), flags
def recv_str(self) -> str:
"""Receive a text frame."""
data, flags = self.recv()
if not (flags & CurlWsFlag.TEXT):
raise WebSocketError("Not valid text frame", WsCloseCode.INVALID_DATA)
return data.decode()
def recv_json(self, *, loads: Callable[[str], T] = loads) -> T:
"""Receive a JSON frame.
Args:
loads: JSON decoder, default is json.loads.
"""
data = self.recv_str()
return loads(data)
def send(self, payload: Union[str, bytes], flags: CurlWsFlag = CurlWsFlag.BINARY):
"""Send a data frame.
Args:
payload: data to send.
flags: flags for the frame.
"""
if flags & CurlWsFlag.CLOSE:
self.keep_running = False
if self.closed:
raise WebSocketClosed("WebSocket is already closed")
# curl expects bytes
if isinstance(payload, str):
payload = payload.encode()
sock_fd = self.curl.getinfo(CurlInfo.ACTIVESOCKET)
if sock_fd == CURL_SOCKET_BAD:
raise WebSocketError(
"Invalid active socket", CurlECode.NO_CONNECTION_AVAILABLE
)
# Loop checks for CurlECode.Again
# https://curl.se/libcurl/c/curl_ws_send.html
offset = 0
while offset < len(payload):
current_buffer = payload[offset:]
try:
n_sent = self.curl.ws_send(current_buffer, flags)
except CurlError as e:
if e.code == CurlECode.AGAIN:
_, writeable, _ = select([], [sock_fd], [], 0.5)
if not writeable:
raise WebSocketError("Socket write timeout") from e
continue
raise
offset += n_sent
return offset
def send_binary(self, payload: bytes):
"""Send a binary frame.
Args:
payload: binary data to send.
"""
return self.send(payload, CurlWsFlag.BINARY)
def send_bytes(self, payload: bytes):
"""Send a binary frame, alias of :meth:`send_binary`.
Args:
payload: binary data to send.
"""
return self.send(payload, CurlWsFlag.BINARY)
def send_str(self, payload: str):
"""Send a text frame.
Args:
payload: text data to send.
"""
return self.send(payload, CurlWsFlag.TEXT)
def send_json(self, payload: Any, *, dumps: Callable[[Any], str] = dumps):
"""Send a JSON frame.
Args:
payload: data to send.
dumps: JSON encoder, default is json.dumps.
"""
return self.send_str(dumps(payload))
def ping(self, payload: Union[str, bytes]):
"""Send a ping frame.
Args:
payload: data to send.
"""
return self.send(payload, CurlWsFlag.PING)
def run_forever(self, url: str = "", **kwargs):
"""Run the WebSocket forever. See :meth:`connect` for details on parameters.
libcurl automatically handles pings and pongs.
ref: https://curl.se/libcurl/c/libcurl-ws.html
"""
if url:
self.connect(url, **kwargs)
sock_fd = self.curl.getinfo(CurlInfo.ACTIVESOCKET)
if sock_fd == CURL_SOCKET_BAD:
raise WebSocketError(
"Invalid active socket", CurlECode.NO_CONNECTION_AVAILABLE
)
self._emit("open")
# Keep reading the messages and invoke callbacks
# TODO: Reconnect logic
chunks = []
self.keep_running = True
while self.keep_running:
try:
chunk, frame = self.recv_fragment()
flags = frame.flags
self._emit("data", chunk, frame)
chunks.append(chunk)
if not (frame.bytesleft == 0 and flags & CurlWsFlag.CONT == 0):
continue
# Avoid unnecessary computation
if "message" in self._emitters:
# Concatenate collected chunks with the final message
msg = b"".join(chunks)
if (flags & CurlWsFlag.TEXT) and not self.skip_utf8_validation:
try:
msg = msg.decode() # type: ignore
except UnicodeDecodeError as e:
self._close_code = WsCloseCode.INVALID_DATA
self.close(WsCloseCode.INVALID_DATA)
raise WebSocketError(
"Invalid UTF-8", WsCloseCode.INVALID_DATA
) from e
if (flags & CurlWsFlag.BINARY) or (flags & CurlWsFlag.TEXT):
self._emit("message", msg)
chunks = [] # Reset chunks for next message
if flags & CurlWsFlag.CLOSE:
self.keep_running = False
self._emit("close", self._close_code or 0, self._close_reason or "")
except CurlError as e:
if e.code == CurlECode.AGAIN:
_, _, _ = select([sock_fd], [], [], 0.5)
else:
self._emit("error", e)
if not self.closed:
code = WsCloseCode.UNKNOWN
if isinstance(e, WebSocketError):
code = e.code
self.close(code)
raise
def close(self, code: int = WsCloseCode.OK, message: bytes = b""):
"""Close the connection.
Args:
code: close code.
message: close reason.
"""
if self.curl is not_set:
return
# TODO: As per spec, we should wait for the server to close the connection
# But this is not a requirement
msg = self._pack_close_frame(code, message)
self.send(msg, CurlWsFlag.CLOSE)
# The only way to close the connection appears to be curl_easy_cleanup
self.terminate()
class AsyncWebSocket(BaseWebSocket):
"""An async WebSocket implementation using libcurl."""
def __init__(
self,
session: AsyncSession,
curl: Curl,
*,
autoclose: bool = True,
debug: bool = False,
):
super().__init__(curl=curl, autoclose=autoclose, debug=debug)
self.session = session
self._loop: Optional[asyncio.AbstractEventLoop] = None
self._recv_lock = asyncio.Lock()
self._send_lock = asyncio.Lock()
@property
def loop(self):
if self._loop is None:
self._loop = get_selector(asyncio.get_running_loop())
return self._loop
def __aiter__(self) -> Self:
if self.closed:
raise WebSocketClosed("WebSocket has been closed")
return self
async def __anext__(self) -> bytes:
msg, flags = await self.recv()
if flags & CurlWsFlag.CLOSE:
raise StopAsyncIteration
return msg
async def recv_fragment(
self, *, timeout: Optional[float] = None
) -> tuple[bytes, CurlWsFrame]:
"""Receive a single frame as bytes.
Args:
timeout: how many seconds to wait before giving up.
"""
if self.closed:
raise WebSocketClosed("WebSocket is closed")
if self._recv_lock.locked():
raise TypeError("Concurrent call to recv_fragment() is not allowed")
async with self._recv_lock:
try:
chunk, frame = await asyncio.wait_for(
self.loop.run_in_executor(None, self.curl.ws_recv), timeout
)
except asyncio.TimeoutError as e:
raise WebSocketTimeout("WebSocket recv_fragment() timed out") from e
if frame.flags & CurlWsFlag.CLOSE:
try:
code, message = self._close_code, self._close_reason = (
self._unpack_close_frame(chunk)
)
except WebSocketError as e:
# Follow the spec to close the connection
# Errors do not respect autoclose
self._close_code = e.code
await self.close(e.code)
raise
if self.autoclose:
await self.close(code, message.encode())
return chunk, frame
async def recv(self, *, timeout: Optional[float] = None) -> tuple[bytes, int]:
"""
Receive a frame as bytes. libcurl splits frames into fragments, so we have to
collect all the chunks for a frame.
Args:
timeout: how many seconds to wait before giving up.
"""
loop = self.loop
chunks = []
flags = 0
sock_fd = await loop.run_in_executor(
None, self.curl.getinfo, CurlInfo.ACTIVESOCKET
)
if sock_fd == CURL_SOCKET_BAD:
raise WebSocketError(
"Invalid active socket", CurlECode.NO_CONNECTION_AVAILABLE
)
while True:
try:
chunk, frame = await self.recv_fragment(timeout=timeout)
flags = frame.flags
chunks.append(chunk)
if frame.bytesleft == 0 and flags & CurlWsFlag.CONT == 0:
break
except CurlError as e:
if e.code == CurlECode.AGAIN:
await aselect(sock_fd, loop=loop, timeout=timeout)
else:
raise
return b"".join(chunks), flags
async def recv_str(self, *, timeout: Optional[float] = None) -> str:
"""Receive a text frame.
Args:
timeout: how many seconds to wait before giving up.
"""
data, flags = await self.recv(timeout=timeout)
if not (flags & CurlWsFlag.TEXT):
raise WebSocketError("Invalid UTF-8", WsCloseCode.INVALID_DATA)
return data.decode()
async def recv_json(
self, *, loads: Callable[[str], T] = loads, timeout: Optional[float] = None
) -> T:
"""Receive a JSON frame.
Args:
loads: JSON decoder, default is json.loads.
timeout: how many seconds to wait before giving up.
"""
data = await self.recv_str(timeout=timeout)
return loads(data)
async def send(
self, payload: Union[str, bytes], flags: CurlWsFlag = CurlWsFlag.BINARY
):
"""Send a data frame.
Args:
payload: data to send.
flags: flags for the frame.
"""
if self.closed:
raise WebSocketClosed("WebSocket is closed")
# curl expects bytes
if isinstance(payload, str):
payload = payload.encode()
sock_fd = await self.loop.run_in_executor(
None, self.curl.getinfo, CurlInfo.ACTIVESOCKET
)
if sock_fd == CURL_SOCKET_BAD:
raise WebSocketError(
"Invalid active socket", CurlECode.NO_CONNECTION_AVAILABLE
)
# TODO: Why does concurrently sending fail
async with self._send_lock:
offset = 0
# Loop checks for CurlECode.Again
# https://curl.se/libcurl/c/curl_ws_send.html
while offset < len(payload):
current_buffer = payload[offset:]
try:
n_sent = await self.loop.run_in_executor(
None, self.curl.ws_send, current_buffer, flags
)
except CurlError as e:
if e.code == CurlECode.AGAIN:
writeable = await aselect(
sock_fd, mode="write", loop=self.loop, timeout=0.5
)
if not writeable:
raise WebSocketError("Socket write timeout") from e
continue
raise
offset += n_sent
return offset
async def send_binary(self, payload: bytes):
"""Send a binary frame.
Args:
payload: binary data to send.
"""
return await self.send(payload, CurlWsFlag.BINARY)
async def send_bytes(self, payload: bytes):
"""Send a binary frame, alias of :meth:`send_binary`.
Args:
payload: binary data to send.
"""
return await self.send(payload, CurlWsFlag.BINARY)
async def send_str(self, payload: str):
"""Send a text frame.
Args:
payload: text data to send.
"""
return await self.send(payload, CurlWsFlag.TEXT)
async def send_json(self, payload: Any, *, dumps: Callable[[Any], str] = dumps):
"""Send a JSON frame.
Args:
payload: data to send.
dumps: JSON encoder, default is json.dumps.
"""
return await self.send_str(dumps(payload))
async def ping(self, payload: Union[str, bytes]):
"""Send a ping frame.
Args:
payload: data to send.
"""
return await self.send(payload, CurlWsFlag.PING)
async def close(self, code: int = WsCloseCode.OK, message: bytes = b""):
"""Close the connection.
Args:
code: close code.
message: close reason.
"""
# TODO: As per spec, we should wait for the server to close the connection
# But this is not a requirement
msg = self._pack_close_frame(code, message)
await self.send(msg, CurlWsFlag.CLOSE)
# The only way to close the connection appears to be curl_easy_cleanup
self.terminate()
def terminate(self):
"""Terminate the underlying connection."""
super().terminate()
if not self.session._closed:
# WebSocket curls CANNOT be reused
self.session.push_curl(None)
WebServer/AIPython/python/lib/python3.11/site-packages/curl_cffi/utils.py
+16
View File
@@ -0,0 +1,16 @@
import warnings
class CurlCffiWarning(UserWarning, RuntimeWarning):
pass
def config_warnings(on: bool = False):
if on:
warnings.simplefilter("default", category=CurlCffiWarning)
else:
warnings.simplefilter("ignore", category=CurlCffiWarning)
def is_pro():
return False