unexpected behavior of stream=True

[9268]

Python 3.8.3 (default, May 19 2020, 06:50:17) [MSC v.1916 64 bit (AMD64)] :: Anaconda, Inc. on win32

pip list
Package                Version
---------------------- -------------------
alembic                1.4.3
apache-airflow         1.10.12
apispec                1.3.3
argcomplete            1.12.1
attrs                  19.3.0
Babel                  2.8.0
cached-property        1.5.2
cattrs                 1.0.0
certifi                2020.4.5.1
cffi                   1.14.0
chardet                3.0.4
click                  7.1.2
colorama               0.4.4
colorlog               4.0.2
conda                  4.8.3
conda-package-handling 1.7.0
configparser           3.5.3
croniter               0.3.35
cryptography           2.9.2
defusedxml             0.6.0
dill                   0.3.2
dnspython              2.0.0
docutils               0.16
email-validator        1.1.1
Flask                  1.1.2
Flask-Admin            1.5.4
Flask-AppBuilder       2.3.4
Flask-Babel            1.0.0
Flask-Caching          1.3.3
Flask-JWT-Extended     3.24.1
Flask-Login            0.4.1
Flask-OpenID           1.2.5
Flask-SQLAlchemy       2.4.4
flask-swagger          0.2.14
Flask-WTF              0.14.3
funcsigs               1.0.2
future                 0.18.2
gevent                 20.9.0
graphviz               0.14.2
greenlet               0.4.17
grequests              0.6.0
gunicorn               20.0.4
idna                   2.9
iso8601                0.1.13
itsdangerous           1.1.0
Jinja2                 2.11.2
json-merge-patch       0.2
jsonschema             3.2.0
lazy-object-proxy      1.5.1
lockfile               0.12.2
Mako                   1.1.3
Markdown               2.6.11
MarkupSafe             1.1.1
marshmallow            2.21.0
marshmallow-enum       1.5.1
marshmallow-sqlalchemy 0.24.0
menuinst               1.4.16
natsort                7.0.1
Nuitka                 0.6.9.4
numpy                  1.19.2
pandas                 1.1.3
pendulum               1.4.4
pip                    20.0.2
prison                 0.1.3
psutil                 5.7.3
pycosat                0.6.3
pycparser              2.20
Pygments               2.7.2
PyJWT                  1.7.1
pyOpenSSL              19.1.0
pyotp                  2.4.1
pyrsistent             0.17.3
PySocks                1.7.1
python-daemon          2.2.4
python-dateutil        2.8.1
python-editor          1.0.4
python-nvd3            0.15.0
python-slugify         4.0.1
python3-openid         3.2.0
pytz                   2020.1
pytzdata               2020.1
pywin32                227
PyYAML                 5.3.1
requests               2.23.0
requests-toolbelt      0.9.1
ruamel-yaml            0.15.87
setproctitle           1.1.10
setuptools             46.4.0.post20200518
six                    1.14.0
SQLAlchemy             1.3.20
SQLAlchemy-JSONField   0.9.0
SQLAlchemy-Utils       0.36.8
tabulate               0.8.7
tenacity               4.12.0
text-unidecode         1.3
thrift                 0.13.0
tornado                6.0.4
tqdm                   4.46.0
tzlocal                1.5.1
unicodecsv             0.14.1
urllib3                1.25.8
Werkzeug               0.16.1
wheel                  0.34.2
win-inet-pton          1.1.0
wincertstore           0.2
WTForms                2.3.3
zope.deprecation       4.4.0
zope.event             4.5.0
zope.interface         5.1.2

i'm trying to use grequests to check live stream ,i just expect a status_code 200,so i use stream = True and close it
my code like :

class ExInfoAdapter(HTTPAdapter): 
    def send(self, request, **kwargs):
        request.raw_info = request.headers.pop("data", None)
        return super(ExInfoAdapter, self).send(request, **kwargs)

cdnsession = requests.session()
cdnsession.headers.update(self.header)
retries = Retry(total=2, backoff_factor=0, status_forcelist=[500, 502, 503, 504, 404], raise_on_redirect=False)
cdnsession.mount('http://', ExInfoAdapter(max_retries=retries, pool_connections=200, pool_maxsize=400))
cdnsession.mount('https://', ExInfoAdapter(max_retries=retries, pool_connections=200, pool_maxsize=400))

for ...
	greqs.append(grequests.get(
                    "http://{rip}/live/{zuid}.flv".format(rip=ip, zuid=zuid),
                    headers={"Host": host,
                             "data": json.dumps( # this will pop out and never send to server
                                 {"rip": ip, "zuid": zuid, "domain": domain, "type": ctype})},
                    timeout=15,
                    session=cdnsession
                ))
def fqecp(request, exception):
    return [request, exception]

resps = grequests.imap(greqs, stream=True, size=200, exception_handler=fqecp)

for resp in resps:
    if isinstance(resp, list):  # handle err
        rd = json.loads(resp[0].kwargs["headers"].get("data", None))
        gerrs[rd['type']].append(rd)
    else:
        rd = json.loads(resp.request.raw_info)
        if resp.status_code != 200:
            gerrs[rd['type']].append(rd)
            print("non-200 : %s" % resp.status_code)
        else:
            pass
            # print("%s check ok" % resp.url)
        resp.close()

it workd fine and prety faster than just requests,but by checking net usage,it seems like stream=True not effect

last 5 low:

it semms like keep download until close() .
i tried requests in shell with stream=True,code like:

import requests
from requests.adapters import HTTPAdapter
from requests.packages.urllib3.util.retry import Retry

class ExInfoAdapter(HTTPAdapter):
    def send(self, request, **kwargs):
        request.raw_info = request.headers.pop("data", None)
        return super(ExInfoAdapter, self).send(request, **kwargs)

s=requests.Session()
retries = Retry(total=2, backoff_factor=0, status_forcelist=[500, 502, 503, 504, 404], raise_on_redirect=False)
s.mount('http://', ExInfoAdapter(max_retries=retries, pool_connections=200, pool_maxsize=400))
s.mount('https://', ExInfoAdapter(max_retries=retries, pool_connections=200, pool_maxsize=400))

r=s.get("http://116.207.172.73/live/2358590143_1603765853.flv",headers={"Host": "v2.zb.marketing.i.mi.com"},timeout=15,stream=True)
assert r.status_code == 200
r.close()
# this ip and url may unavilable when you try,please find another live stream or call me to get a new one

here 's the result:

i tryedboth grequests and requests 2 times , grequests download avg 3MB(2.8and3.2) and requests download avg 322KB(321and323)
323KB is mach accessable but still a lot,i just want to chek http code ,it's still a lot
where's my fault and how to resolve this issue?